Fix pnpm lock file

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/breezy-gorillas-mate.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+better handle task metadata parse errors, and display nicely formatted errors

.changeset/clean-pianos-listen.md

@@ -0,0 +1,8 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+add machine config and secure zod connection

.changeset/cool-glasses-bake.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Configurable log levels in the config file and via env var

.changeset/green-bags-wink.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/react": patch
+---
+
+Fix for shared queryKey between useRunDetails and useRunStatuses

.changeset/khaki-apricots-design.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Retry 429, 500, and connection error API requests to the trigger.dev server

.changeset/light-bulldogs-press.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Changed "Worker" to "Version" in the dev command key

.changeset/many-ligers-pump.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Handle string and non-stringifiable outputs like functions

.changeset/mighty-camels-joke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improve error messages during dev/deploy and handle deploy image build issues

.changeset/odd-poets-own.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Added a Node.js runtime check for the CLI

.changeset/pre.json

@@ -44,8 +44,22 @@
     "@trigger.dev/yalt": "2.3.18"
   },
   "changesets": [
+    "breezy-gorillas-mate",
     "chilled-hornets-move",
+    "clean-pianos-listen",
+    "cool-glasses-bake",
+    "green-bags-wink",
+    "khaki-apricots-design",
     "lemon-jobs-repair",
-    "sweet-lizards-press"
+    "light-bulldogs-press",
+    "many-ligers-pump",
+    "mighty-camels-joke",
+    "odd-poets-own",
+    "real-planets-stare",
+    "smart-olives-eat",
+    "strange-ghosts-matter",
+    "stupid-bulldogs-applaud",
+    "sweet-lizards-press",
+    "tall-bees-wave"
   ]
 }

.changeset/real-planets-stare.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fixed incorrect span timings around checkpoints by implementing a precise wall clock that resets after restores

.changeset/smart-olives-eat.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix CLI logout and add list-profiles command

.changeset/strange-ghosts-matter.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Added DEBUG to the ignored env vars

.changeset/stupid-bulldogs-applaud.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixed an issue where the trigger.dev package was not being built before publishing to npm

.changeset/tall-bees-wave.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Dynamically import superjson and fix some bundling issues

.env.example

@@ -63,4 +63,13 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # These are needed for the object store (for handling large payloads/outputs)
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
 # OBJECT_STORE_ACCESS_KEY_ID=
-# OBJECT_STORE_SECRET_ACCESS_KEY=
+# OBJECT_STORE_SECRET_ACCESS_KEY=
+# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+
+# These control the server-side internal telemetry
+# INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME=<Header name for the auth token>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
+# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
+# INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,

apps/coordinator/.env.example

@@ -1,3 +1,4 @@
 HTTP_SERVER_PORT=8020
 PLATFORM_ENABLED=true
-PLATFORM_WS_PORT=3030
+PLATFORM_WS_PORT=3030
+SECURE_CONNECTION=false

apps/coordinator/src/index.ts

@@ -18,6 +18,8 @@ collectDefaultMetrics();
 
 const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020);
 const NODE_NAME = process.env.NODE_NAME || "coordinator";
+const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000;
+
 const REGISTRY_HOST = process.env.REGISTRY_HOST || "localhost:5000";
 const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints";
 const REGISTRY_TLS_VERIFY = process.env.REGISTRY_TLS_VERIFY === "false" ? "false" : "true";
@@ -26,6 +28,7 @@ const PLATFORM_ENABLED = ["1", "true"].includes(process.env.PLATFORM_ENABLED ??
 const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
 const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
 const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret";
+const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
 
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 
@@ -333,12 +336,19 @@ class TaskCoordinator {
     { resolve: (value: void) => void; reject: (err?: any) => void }
   >();
 
+  #delayThresholdInMs: number;
+
   constructor(
     private port: number,
     private host = "0.0.0.0"
   ) {
     this.#httpServer = this.#createHttpServer();
     this.#checkpointer.initialize();
+    this.#delayThresholdInMs = this.#getDelayThreshold();
+
+    if (process.env.DELAY_THRESHOLD_IN_MS) {
+      this.#delayThresholdInMs = this.#getDelayThreshold();
+    }
 
     const io = new Server(this.#httpServer);
     this.#prodWorkerNamespace = this.#createProdWorkerNamespace(io);
@@ -355,6 +365,27 @@ class TaskCoordinator {
     register.registerMetric(connectedTasksTotal);
   }
 
+  #getDelayThreshold() {
+    if (!process.env.RETRY_DELAY_THRESHOLD_IN_MS) {
+      return DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS;
+    }
+
+    const threshold = parseInt(process.env.RETRY_DELAY_THRESHOLD_IN_MS);
+
+    if (isNaN(threshold)) {
+      logger.log(
+        "RETRY_DELAY_THRESHOLD_IN_MS parses as NaN, must supply integer. Will use default instead.",
+        {
+          RETRY_DELAY_THRESHOLD_IN_MS: process.env.RETRY_DELAY_THRESHOLD_IN_MS,
+          DEFAULT_DELAY_THRESHOLD_IN_MS: DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS,
+        }
+      );
+      return DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS;
+    }
+
+    return threshold;
+  }
+
   #createPlatformSocket() {
     if (!PLATFORM_ENABLED) {
       console.log("INFO: platform connection disabled");
@@ -365,6 +396,7 @@ class TaskCoordinator {
       namespace: "coordinator",
       host: PLATFORM_HOST,
       port: Number(PLATFORM_WS_PORT),
+      secure: SECURE_CONNECTION,
       clientMessages: CoordinatorToPlatformMessages,
       serverMessages: PlatformToCoordinatorMessages,
       authToken: PLATFORM_SECRET,
@@ -627,7 +659,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (completion.retry.delay < 10_000) {
+          if (completion.retry.delay < this.#delayThresholdInMs) {
             completeWithoutCheckpoint(false);
             return;
           }

apps/docker-provider/.env.example

@@ -2,6 +2,7 @@ HTTP_SERVER_PORT=8050
 
 PLATFORM_WS_PORT=3030
 PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
 
 # Use this if you are on macOS
 # COORDINATOR_HOST="host.docker.internal"

apps/docker-provider/src/index.ts

@@ -8,6 +8,7 @@ import {
   TaskOperationsIndexOptions,
 } from "@trigger.dev/core-apps";
 import { setTimeout } from "node:timers/promises";
+import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
 
 const MACHINE_NAME = process.env.MACHINE_NAME || "local";
 const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
@@ -190,6 +191,9 @@ class DockerTaskOperations implements TaskOperations {
   async delete(opts: { runId: string }) {
     await this.#initialize();
 
+    const containerName = this.#getRunContainerName(opts.runId);
+    await this.#sendPreStop(containerName);
+
     logger.log("noop: delete");
   }
 
@@ -208,6 +212,26 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async #sendPostStart(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
+    } catch (error) {
+      logger.error("postStart error", { error });
+      throw new Error("postStart command failed");
+    }
+  }
+
+  async #sendPreStop(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate"));
+    } catch (error) {
+      logger.error("preStop error", { error });
+      throw new Error("preStop command failed");
+    }
+  }
+
+  async #getHttpServerPort(containerName: string): Promise<number> {
     // We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes
     // FIXME: Skip this in prod
     const logs = logger.debug(await $`docker logs ${containerName}`);
@@ -219,19 +243,14 @@ class DockerTaskOperations implements TaskOperations {
       throw new Error("failed to extract port from logs");
     }
 
-    try {
-      logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
-    } catch (error) {
-      logger.error("postStart error", { error });
-      throw new Error("postStart command failed");
-    }
+    return port;
   }
 
-  async #runLifecycleCommand(
+  async #runLifecycleCommand<THookType extends "postStart" | "preStop">(
     containerName: string,
     port: number,
-    type: "postStart" | "preStop",
-    cause: "index" | "create" | "restore",
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses,
     retryCount = 0
   ): Promise<ExecaChildProcess> {
     try {
@@ -244,15 +263,15 @@ class DockerTaskOperations implements TaskOperations {
         `127.0.0.1:${port}/${type}?cause=${cause}`,
       ]);
     } catch (error: any) {
-      if (retryCount < 6) {
-        logger.debug("retriable postStart error", { retryCount, message: error?.message });
+      if (type === "postStart" && retryCount < 6) {
+        logger.debug(`retriable ${type} error`, { retryCount, message: error?.message });
         await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50));
 
         return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1);
       }
 
-      logger.error("final postStart error", { message: error?.message });
-      throw new Error(`postStart command failed after ${retryCount - 1} retries`);
+      logger.error(`final ${type} error`, { message: error?.message });
+      throw new Error(`${type} command failed after ${retryCount - 1} retries`);
     }
   }
 }

apps/kubernetes-provider/.env.example

@@ -2,6 +2,7 @@ HTTP_SERVER_PORT=8060
 
 PLATFORM_WS_PORT=3030
 PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
 
 # Use this if you are on macOS
 # COORDINATOR_HOST="host.docker.internal"

apps/kubernetes-provider/src/index.ts

@@ -7,6 +7,7 @@ import {
   TaskOperationsIndexOptions,
   TaskOperationsRestoreOptions,
 } from "@trigger.dev/core-apps";
+import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
@@ -55,6 +56,12 @@ class KubernetesTaskOperations implements TaskOperations {
             metadata: {
               labels: {
                 app: "task-index",
+                "app.kubernetes.io/part-of": "trigger-worker",
+                "app.kubernetes.io/component": "index",
+                env: opts.envId,
+                envtype: this.#envTypeToLabelValue(opts.envType),
+                org: opts.orgId,
+                project: opts.projectId,
               },
             },
             spec: {
@@ -64,6 +71,9 @@ class KubernetesTaskOperations implements TaskOperations {
                   name: "registry-trigger",
                 },
               ],
+              nodeSelector: {
+                nodetype: "worker",
+              },
               containers: [
                 {
                   name: this.#getIndexContainerName(opts.shortCode),
@@ -79,6 +89,13 @@ class KubernetesTaskOperations implements TaskOperations {
                   //     memory: "50Mi",
                   //   },
                   // },
+                  lifecycle: {
+                    preStop: {
+                      exec: {
+                        command: this.#getLifecycleCommand("preStop", "terminate"),
+                      },
+                    },
+                  },
                   env: [
                     {
                       name: "DEBUG",
@@ -151,6 +168,13 @@ class KubernetesTaskOperations implements TaskOperations {
           namespace: this.#namespace.metadata.name,
           labels: {
             app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "create",
+            env: opts.envId,
+            envtype: this.#envTypeToLabelValue(opts.envType),
+            org: opts.orgId,
+            project: opts.projectId,
+            run: opts.runId,
           },
         },
         spec: {
@@ -160,6 +184,9 @@ class KubernetesTaskOperations implements TaskOperations {
               name: "registry-trigger",
             },
           ],
+          nodeSelector: {
+            nodetype: "worker",
+          },
           containers: [
             {
               name: this.#getRunContainerName(opts.runId),
@@ -169,9 +196,9 @@ class KubernetesTaskOperations implements TaskOperations {
                   containerPort: 8000,
                 },
               ],
-              // resources: {
-              //   limits: opts.machine,
-              // },
+              resources: {
+                limits: this.#getResourcesFromMachineConfig(opts.machine),
+              },
               lifecycle: {
                 postStart: {
                   exec: {
@@ -180,7 +207,7 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
                 preStop: {
                   exec: {
-                    command: this.#getLifecycleCommand("preStop", "create"),
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
                   },
                 },
               },
@@ -262,6 +289,14 @@ class KubernetesTaskOperations implements TaskOperations {
           namespace: this.#namespace.metadata.name,
           labels: {
             app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "restore",
+            env: opts.envId,
+            envtype: this.#envTypeToLabelValue(opts.envType),
+            org: opts.orgId,
+            project: opts.projectId,
+            run: opts.runId,
+            checkpoint: opts.checkpointId,
           },
         },
         spec: {
@@ -271,6 +306,9 @@ class KubernetesTaskOperations implements TaskOperations {
               name: "registry-trigger",
             },
           ],
+          nodeSelector: {
+            nodetype: "worker",
+          },
           initContainers: [
             {
               name: "pull-base-image",
@@ -309,9 +347,9 @@ class KubernetesTaskOperations implements TaskOperations {
                   containerPort: 8000,
                 },
               ],
-              // resources: {
-              //   limits: opts.machine,
-              // },
+              resources: {
+                limits: this.#getResourcesFromMachineConfig(opts.machine),
+              },
               lifecycle: {
                 postStart: {
                   exec: {
@@ -320,7 +358,7 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
                 preStop: {
                   exec: {
-                    command: this.#getLifecycleCommand("preStop", "restore"),
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
                   },
                 },
               },
@@ -355,7 +393,30 @@ class KubernetesTaskOperations implements TaskOperations {
     await this.#getPod(opts.runId, this.#namespace);
   }
 
-  #getLifecycleCommand(type: "postStart" | "preStop", cause: "index" | "create" | "restore") {
+  #envTypeToLabelValue(type: EnvironmentType) {
+    switch (type) {
+      case "PRODUCTION":
+        return "prod";
+      case "STAGING":
+        return "stg";
+      case "DEVELOPMENT":
+        return "dev";
+      case "PREVIEW":
+        return "preview";
+    }
+  }
+
+  #getResourcesFromMachineConfig(config: Machine) {
+    return {
+      cpu: `${config.cpu}`,
+      memory: `${config.memory}G`,
+    };
+  }
+
+  #getLifecycleCommand<THookType extends "postStart" | "preStop">(
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
+  ) {
     return ["/bin/sh", "-c", `sleep 1; wget -q -O- 127.0.0.1:8000/${type}?cause=${cause}`];
   }
 

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -264,7 +264,7 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
             <SideMenuItem
               name="Documentation (v3)"
               icon="docs"
-              to="https://trigger.dev/docs"
+              to="https://trigger.dev/docs/v3"
               data-action="documentation"
               target="_blank"
             />
@@ -556,7 +556,7 @@ function V3ProjectSideMenu({
         data-action="test"
       />
       <SideMenuItem
-        name="API Keys"
+        name="API keys"
         icon={KeyIcon}
         iconColor="text-amber-500"
         to={v3ApiKeysPath(organization, project)}

apps/webapp/app/entry.server.tsx

@@ -14,9 +14,9 @@ import {
   OperatingSystemContextProvider,
   OperatingSystemPlatform,
 } from "./components/primitives/OperatingSystemProvider";
-import { env } from "./env.server";
 import { getSharedSqsEventConsumer } from "./services/events/sqsEventConsumer";
 import { singleton } from "./utils/singleton";
+import { logger } from "./services/logger.server";
 
 const ABORT_DELAY = 30000;
 
@@ -178,7 +178,7 @@ function logError(error: unknown, request?: Request) {
 
 const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer);
 
-export { wss } from "./v3/handleWebsockets.server";
-export { socketIo } from "./v3/handleSocketIo.server";
-export { registryProxy } from "./v3/registryProxy.server";
 export { apiRateLimiter } from "./services/apiRateLimit.server";
+export { socketIo } from "./v3/handleSocketIo.server";
+export { wss } from "./v3/handleWebsockets.server";
+export { registryProxy } from "./v3/registryProxy.server";

apps/webapp/app/env.server.ts

@@ -19,6 +19,7 @@ const EnvironmentSchema = z.object({
   LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ENV: z.string().default(process.env.NODE_ENV),
+  SERVICE_NAME: z.string().default("trigger.dev webapp"),
   SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"),
   POSTHOG_PROJECT_KEY: z.string().optional(),
   TELEMETRY_TRIGGER_API_KEY: z.string().optional(),
@@ -63,6 +64,8 @@ const EnvironmentSchema = z.object({
   REDIS_PASSWORD: z.string().optional(),
   REDIS_TLS_DISABLED: z.string().optional(),
 
+  DEFAULT_QUEUE_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(5),
+  DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10),
   DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10),
   DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1),
 
@@ -85,10 +88,6 @@ const EnvironmentSchema = z.object({
 
   //v3
   V3_ENABLED: z.string().default("false"),
-  OTLP_EXPORTER_TRACES_URL: z.string().optional(),
-  LOG_TELEMETRY: z.string().default("true"),
-  IMAGE_REGISTRY: z.string().default("docker.io"),
-  IMAGE_REPO: z.string().default("task"),
   PROVIDER_SECRET: z.string().default("provider-secret"),
   COORDINATOR_SECRET: z.string().default("coordinator-secret"),
   DEPOT_TOKEN: z.string().optional(),
@@ -97,13 +96,38 @@ const EnvironmentSchema = z.object({
   CONTAINER_REGISTRY_USERNAME: z.string().optional(),
   CONTAINER_REGISTRY_PASSWORD: z.string().optional(),
   DEPLOY_REGISTRY_HOST: z.string().optional(),
-  DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
   OBJECT_STORE_BASE_URL: z.string().optional(),
   OBJECT_STORE_ACCESS_KEY_ID: z.string().optional(),
   OBJECT_STORE_SECRET_ACCESS_KEY: z.string().optional(),
   EVENTS_BATCH_SIZE: z.coerce.number().int().default(100),
   EVENTS_BATCH_INTERVAL: z.coerce.number().int().default(1000),
   EVENTS_DEFAULT_LOG_RETENTION: z.coerce.number().int().default(7),
+  SHARED_QUEUE_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10),
+  SHARED_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(100),
+  SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS: z.coerce.number().int().default(100),
+
+  // Development OTEL environment variables
+  DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
+  // If this is set to 1, then the below variables are used to configure the batch processor for spans and logs
+  DEV_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"),
+  DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
+  DEV_OTEL_SPAN_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
+  DEV_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
+  DEV_OTEL_SPAN_MAX_QUEUE_SIZE: z.string().default("512"),
+  DEV_OTEL_LOG_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
+  DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
+  DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
+  DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
+  RUNTIME_WAIT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000),
+
+  // Internal OTEL environment variables
+  INTERNAL_OTEL_TRACE_EXPORTER_URL: z.string().optional(),
+  INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME: z.string().optional(),
+  INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE: z.string().optional(),
+  INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"),
+  // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+  INTERNAL_OTEL_TRACE_SAMPING_RATE: z.string().default("20"),
+  INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -1,8 +1,15 @@
+import {
+  DeploymentErrorData,
+  TaskMetadataFailedToParseData,
+  groupTaskMetadataIssuesByTask,
+} from "@trigger.dev/core/v3";
 import { WorkerDeployment, WorkerDeploymentStatus } from "@trigger.dev/database";
+import { z } from "zod";
 import { PrismaClient, prisma } from "~/db.server";
 import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
+import { safeJsonParse } from "~/utils/json";
 import { getUsername } from "~/utils/username";
 
 export class DeploymentPresenter {
@@ -51,6 +58,7 @@ export class DeploymentPresenter {
         id: true,
         shortCode: true,
         version: true,
+        errorData: true,
         environment: {
           select: {
             id: true,
@@ -120,7 +128,81 @@ export class DeploymentPresenter {
           userName: getUsername(deployment.environment.orgMember?.user),
         },
         deployedBy: deployment.triggeredBy,
+        errorData: this.#prepareErrorData(deployment.errorData),
       },
     };
   }
+
+  #prepareErrorData(errorData: WorkerDeployment["errorData"]) {
+    if (!errorData) {
+      return;
+    }
+
+    const parsedErrorData = DeploymentErrorData.safeParse(errorData);
+
+    if (!parsedErrorData.success) {
+      return;
+    }
+
+    if (parsedErrorData.data.name === "TaskMetadataParseError") {
+      const errorJson = safeJsonParse(parsedErrorData.data.stack);
+
+      if (errorJson) {
+        const parsedError = TaskMetadataFailedToParseData.safeParse(errorJson);
+
+        if (parsedError.success) {
+          return {
+            name: parsedErrorData.data.name,
+            message: parsedErrorData.data.message,
+            stack: createTaskMetadataFailedErrorStack(parsedError.data),
+          };
+        } else {
+          return {
+            name: parsedErrorData.data.name,
+            message: parsedErrorData.data.message,
+          };
+        }
+      } else {
+        return {
+          name: parsedErrorData.data.name,
+          message: parsedErrorData.data.message,
+        };
+      }
+    }
+
+    return {
+      name: parsedErrorData.data.name,
+      message: parsedErrorData.data.message,
+      stack: parsedErrorData.data.stack,
+    };
+  }
+}
+
+function createTaskMetadataFailedErrorStack(
+  data: z.infer<typeof TaskMetadataFailedToParseData>
+): string {
+  const stack = [];
+
+  const groupedIssues = groupTaskMetadataIssuesByTask(data.tasks, data.zodIssues);
+
+  for (const key in groupedIssues) {
+    const taskWithIssues = groupedIssues[key];
+
+    if (!taskWithIssues) {
+      continue;
+    }
+
+    stack.push("\n");
+    stack.push(`  ❯ ${taskWithIssues.exportName} in ${taskWithIssues.filePath}`);
+
+    for (const issue of taskWithIssues.issues) {
+      if (issue.path) {
+        stack.push(`    x ${issue.path} ${issue.message}`);
+      } else {
+        stack.push(`    x ${issue.message}`);
+      }
+    }
+  }
+
+  return stack.join("\n");
 }

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -43,14 +43,14 @@ export class SpanPresenter {
       span.outputType === "application/store"
         ? `/resources/packets/${span.environmentId}/${span.output}`
         : typeof span.output !== "undefined" && span.output !== null
-        ? prettyPrintPacket(span.output, span.outputType ?? undefined)
+        ? await prettyPrintPacket(span.output, span.outputType ?? undefined)
         : undefined;
 
     const payload =
       span.payloadType === "application/store"
         ? `/resources/packets/${span.environmentId}/${span.payload}`
         : typeof span.payload !== "undefined" && span.payload !== null
-        ? prettyPrintPacket(span.payload, span.payloadType ?? undefined)
+        ? await prettyPrintPacket(span.payload, span.payloadType ?? undefined)
         : undefined;
 
     return {

apps/webapp/app/presenters/v3/TestTaskPresenter.server.ts

@@ -69,7 +69,8 @@ export class TestTaskPresenter {
       ON
           tr."taskIdentifier" = bwt.slug
       WHERE
-          bwt."friendlyId" = ${taskFriendlyId}
+          bwt."friendlyId" = ${taskFriendlyId} AND
+          tr."runtimeEnvironmentId" = ${task.runtimeEnvironment.id}
       ORDER BY 
           tr."createdAt" DESC
       LIMIT 5

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.apikeys/route.tsx

@@ -19,6 +19,7 @@ import {
   TableHeaderCell,
   TableRow,
 } from "~/components/primitives/Table";
+import { TextLink } from "~/components/primitives/TextLink";
 import { useProject } from "~/hooks/useProject";
 import { ApiKeysPresenter } from "~/presenters/v3/ApiKeysPresenter.server";
 import { requireUserId } from "~/services/session.server";
@@ -55,12 +56,12 @@ export default function Page() {
   return (
     <PageContainer>
       <NavBar>
-        <PageTitle title="API Keys" />
+        <PageTitle title="API keys" />
         <PageAccessories>
           <LinkButton
             variant={"minimal/small"}
             LeadingIcon={BookOpenIcon}
-            to={docsPath("/documentation/concepts/environments-endpoints#environments")}
+            to={docsPath("/v3/apikeys")}
           >
             API keys docs
           </LinkButton>
@@ -68,9 +69,10 @@ export default function Page() {
       </NavBar>
       <PageBody>
         <div className={cn("h-full")}>
-          <Header3 spacing>Server API keys</Header3>
+          <Header3 spacing>Secret keys</Header3>
           <Paragraph variant="small" spacing>
-            Server API keys should be used on your server – they give full API access.
+            Secret keys should be used on your server – they give full API access and allow you to{" "}
+            <TextLink to={docsPath("v3/triggering")}>trigger tasks</TextLink> from your backend.
           </Paragraph>
           <Header3 spacing>Public API keys</Header3>
           <Paragraph variant="small" spacing>
@@ -81,7 +83,7 @@ export default function Page() {
               <TableHeader>
                 <TableRow>
                   <TableHeaderCell>Environment</TableHeaderCell>
-                  <TableHeaderCell>Server API key</TableHeaderCell>
+                  <TableHeaderCell>Secret key</TableHeaderCell>
                   <TableHeaderCell>Public API key</TableHeaderCell>
                   <TableHeaderCell>Keys generated</TableHeaderCell>
                   <TableHeaderCell>Latest version</TableHeaderCell>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx

@@ -3,6 +3,7 @@ import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
 import { UserAvatar } from "~/components/UserProfilePhoto";
+import { CodeBlock } from "~/components/code/CodeBlock";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { Badge } from "~/components/primitives/Badge";
 import { LinkButton } from "~/components/primitives/Buttons";
@@ -156,6 +157,26 @@ export default function Page() {
                 </TableBody>
               </Table>
             </div>
+          ) : deployment.errorData ? (
+            <div className="flex flex-col">
+              {deployment.errorData.stack ? (
+                <CodeBlock
+                  language="markdown"
+                  rowTitle={deployment.errorData.message}
+                  code={deployment.errorData.stack}
+                  maxLines={20}
+                />
+              ) : (
+                <div className="flex flex-col">
+                  <Paragraph
+                    variant="base/bright"
+                    className="w-full border-b border-grid-dimmed py-2.5"
+                  >
+                    {deployment.errorData.message}
+                  </Paragraph>
+                </div>
+              )}
+            </div>
           ) : null}
         </div>
       </div>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables.new/route.tsx

@@ -185,7 +185,11 @@ export default function Page() {
                       >
                         <EnvironmentLabel environment={environment} className="h-5 px-2" />
                       </label>
-                      <Input name={`values[${index}].value`} placeholder="Not set" />
+                      <Input
+                        type="password"
+                        name={`values[${index}].value`}
+                        placeholder="Not set"
+                      />
                     </Fragment>
                   );
                 })}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables/route.tsx

@@ -167,7 +167,7 @@ export default function Page() {
         <PageAccessories>
           <LinkButton
             LeadingIcon={BookOpenIcon}
-            to={docsPath("/documentation/concepts/environments-endpoints#environments")}
+            to={docsPath("v3/deploy-environment-variables")}
             variant="minimal/small"
           >
             Environment variables docs

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -222,19 +222,25 @@ function PacketDisplay({
   dataType: string;
   title: string;
 }) {
-  if (dataType === "application/store") {
-    return (
-      <div className="flex flex-col">
-        <Paragraph variant="base/bright" className="w-full border-b border-grid-dimmed py-2.5">
-          {title}
-        </Paragraph>
-        <LinkButton LeadingIcon={CloudArrowDownIcon} to={data} variant="tertiary/medium" download>
-          Download
-        </LinkButton>
-      </div>
-    );
-  } else {
-    return <CodeBlock rowTitle={title} code={data} maxLines={20} />;
+  switch (dataType) {
+    case "application/store": {
+      return (
+        <div className="flex flex-col">
+          <Paragraph variant="base/bright" className="w-full border-b border-grid-dimmed py-2.5">
+            {title}
+          </Paragraph>
+          <LinkButton LeadingIcon={CloudArrowDownIcon} to={data} variant="tertiary/medium" download>
+            Download
+          </LinkButton>
+        </div>
+      );
+    }
+    case "text/plain": {
+      return <CodeBlock language="markdown" rowTitle={title} code={data} maxLines={20} />;
+    }
+    default: {
+      return <CodeBlock language="json" rowTitle={title} code={data} maxLines={20} />;
+    }
   }
 }
 

apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts

@@ -0,0 +1,66 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { marqs } from "~/v3/marqs/index.server";
+
+const ParamsSchema = z.object({
+  environmentId: z.string(),
+});
+
+const RequestBodySchema = z.object({
+  envMaximumConcurrencyLimit: z.number(),
+  orgMaximumConcurrencyLimit: z.number(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  const parsedParams = ParamsSchema.parse(params);
+
+  const rawBody = await request.json();
+  const body = RequestBodySchema.parse(rawBody);
+
+  const environment = await prisma.runtimeEnvironment.update({
+    where: {
+      id: parsedParams.environmentId,
+    },
+    data: {
+      maximumConcurrencyLimit: body.envMaximumConcurrencyLimit,
+      organization: {
+        update: {
+          data: {
+            maximumConcurrencyLimit: body.orgMaximumConcurrencyLimit,
+          },
+        },
+      },
+    },
+    include: {
+      organization: true,
+      project: true,
+    },
+  });
+
+  await marqs?.updateEnvConcurrencyLimits(environment);
+
+  return json({ success: true });
+}

apps/webapp/app/utils/json.ts

@@ -1,6 +1,10 @@
 import { z } from "zod";
 
-export function safeJsonParse(json: string): unknown {
+export function safeJsonParse(json?: string): unknown {
+  if (!json) {
+    return;
+  }
+
   try {
     return JSON.parse(json);
   } catch (e) {

apps/webapp/app/utils/throttle.ts

@@ -1,20 +1,20 @@
 //From: https://kettanaito.com/blog/debounce-vs-throttle
 
-/** A very simple throttle. Will execute the function every Xms and discard any other calls during that period. */
+/** A very simple throttle. Will execute the function at the end of each period and discard any other calls during that period. */
 export function throttle(
   func: (...args: any[]) => void,
-  duration: number
+  durationMs: number
 ): (...args: any[]) => void {
-  let shouldWait = false;
+  let isPrimedToFire = false;
 
   return (...args: any[]) => {
-    if (!shouldWait) {
-      func(...args);
-      shouldWait = true;
+    if (!isPrimedToFire) {
+      isPrimedToFire = true;
 
       setTimeout(() => {
-        shouldWait = false;
-      }, duration);
+        func(...args);
+        isPrimedToFire = false;
+      }, durationMs);
     }
   };
 }

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -30,7 +30,7 @@ function parseSecretKey(key: string) {
 const SecretValue = z.object({ secret: z.string() });
 
 export class EnvironmentVariablesRepository implements Repository {
-  constructor(private prismaClient: PrismaClient = prisma) { }
+  constructor(private prismaClient: PrismaClient = prisma) {}
 
   async create(
     projectId: string,
@@ -419,8 +419,49 @@ export class EnvironmentVariablesRepository implements Repository {
         {
           key: "OTEL_EXPORTER_OTLP_ENDPOINT",
           value: env.DEV_OTEL_EXPORTER_OTLP_ENDPOINT ?? env.APP_ORIGIN,
-        }
-      ];
+        },
+      ].concat(
+        env.DEV_OTEL_BATCH_PROCESSING_ENABLED === "1"
+          ? [
+              {
+                key: "OTEL_BATCH_PROCESSING_ENABLED",
+                value: "1",
+              },
+              {
+                key: "OTEL_SPAN_MAX_EXPORT_BATCH_SIZE",
+                value: env.DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE,
+              },
+              {
+                key: "OTEL_SPAN_SCHEDULED_DELAY_MILLIS",
+                value: env.DEV_OTEL_SPAN_SCHEDULED_DELAY_MILLIS,
+              },
+              {
+                key: "OTEL_SPAN_EXPORT_TIMEOUT_MILLIS",
+                value: env.DEV_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS,
+              },
+              {
+                key: "OTEL_SPAN_MAX_QUEUE_SIZE",
+                value: env.DEV_OTEL_SPAN_MAX_QUEUE_SIZE,
+              },
+              {
+                key: "OTEL_LOG_MAX_EXPORT_BATCH_SIZE",
+                value: env.DEV_OTEL_LOG_MAX_EXPORT_BATCH_SIZE,
+              },
+              {
+                key: "OTEL_LOG_SCHEDULED_DELAY_MILLIS",
+                value: env.DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS,
+              },
+              {
+                key: "OTEL_LOG_EXPORT_TIMEOUT_MILLIS",
+                value: env.DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS,
+              },
+              {
+                key: "OTEL_LOG_MAX_QUEUE_SIZE",
+                value: env.DEV_OTEL_LOG_MAX_QUEUE_SIZE,
+              },
+            ]
+          : []
+      );
     }
 
     return [
@@ -432,6 +473,10 @@ export class EnvironmentVariablesRepository implements Repository {
         key: "TRIGGER_API_URL",
         value: env.APP_ORIGIN,
       },
+      {
+        key: "TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS",
+        value: String(env.RUNTIME_WAIT_THRESHOLD_IN_MS),
+      },
     ];
   }
 

apps/webapp/app/v3/eventRepository.server.ts

@@ -10,7 +10,7 @@ import {
   SpanMessagingEvent,
   TaskEventStyle,
   correctErrorStackTrace,
-  createPackageAttributesAsJson,
+  createPacketAttributesAsJson,
   flattenAttributes,
   isExceptionSpanEvent,
   omit,
@@ -188,7 +188,7 @@ export class EventRepository {
     const event = events[0];
 
     const output = options?.attributes.output
-      ? createPackageAttributesAsJson(
+      ? await createPacketAttributesAsJson(
           options?.attributes.output,
           options?.attributes.outputType ?? "application/json"
         )
@@ -213,8 +213,9 @@ export class EventRepository {
       style: event.style as Attributes,
       output: output,
       outputType:
-        options?.attributes.outputType === "application/store"
-          ? "application/store"
+        options?.attributes.outputType === "application/store" ||
+        options?.attributes.outputType === "text/plain"
+          ? options?.attributes.outputType
           : "application/json",
       payload: event.payload as Attributes,
       payloadType: event.payloadType,

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -168,6 +168,7 @@ function createSharedQueueConsumerNamespace(io: Server) {
         namespace: sharedQueue.namespace,
         socket,
         logger,
+        poolSize: env.SHARED_QUEUE_CONSUMER_POOL_SIZE,
       });
 
       sharedSocketConnection.onClose.attach((closeEvent) => {

apps/webapp/app/v3/marqs/devQueueConsumer.server.ts

@@ -13,7 +13,7 @@ import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { CancelAttemptService } from "../services/cancelAttempt.server";
 import { CompleteAttemptService } from "../services/completeAttempt.server";
 import { attributesFromAuthenticatedEnv } from "../tracer.server";

apps/webapp/app/v3/marqs/marqsKeyProducer.server.ts

@@ -0,0 +1,119 @@
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { MarQSKeyProducer } from "./types";
+
+const constants = {
+  SHARED_QUEUE: "sharedQueue",
+  CURRENT_CONCURRENCY_PART: "currentConcurrency",
+  CONCURRENCY_LIMIT_PART: "concurrency",
+  ENV_PART: "env",
+  ORG_PART: "org",
+  QUEUE_PART: "queue",
+  CONCURRENCY_KEY_PART: "ck",
+  MESSAGE_PART: "message",
+} as const;
+
+export class MarQSShortKeyProducer implements MarQSKeyProducer {
+  constructor(private _prefix: string) {}
+
+  queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string) {
+    return [this.queueKey(env, queue), constants.CONCURRENCY_LIMIT_PART].join(":");
+  }
+
+  envConcurrencyLimitKey(env: AuthenticatedEnvironment) {
+    return [this.envKeySection(env.id), constants.CONCURRENCY_LIMIT_PART].join(":");
+  }
+
+  orgConcurrencyLimitKey(env: AuthenticatedEnvironment) {
+    return [this.orgKeySection(env.organizationId), constants.CONCURRENCY_LIMIT_PART].join(":");
+  }
+
+  queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string) {
+    return [
+      this.orgKeySection(env.organizationId),
+      this.envKeySection(env.id),
+      this.queueSection(queue),
+    ]
+      .concat(concurrencyKey ? this.concurrencyKeySection(concurrencyKey) : [])
+      .join(":");
+  }
+
+  envSharedQueueKey(env: AuthenticatedEnvironment) {
+    if (env.type === "DEVELOPMENT") {
+      return [
+        this.orgKeySection(env.organizationId),
+        this.envKeySection(env.id),
+        constants.SHARED_QUEUE,
+      ].join(":");
+    }
+
+    return constants.SHARED_QUEUE;
+  }
+
+  concurrencyLimitKeyFromQueue(queue: string) {
+    const concurrencyQueueName = queue.replace(/:ck:.+$/, "");
+
+    return `${concurrencyQueueName}:${constants.CONCURRENCY_LIMIT_PART}`;
+  }
+
+  currentConcurrencyKeyFromQueue(queue: string) {
+    return `${queue}:${constants.CURRENT_CONCURRENCY_PART}`;
+  }
+
+  orgConcurrencyLimitKeyFromQueue(queue: string) {
+    const orgId = this.normalizeQueue(queue).split(":")[1];
+
+    return `${constants.ORG_PART}:${orgId}:${constants.CONCURRENCY_LIMIT_PART}`;
+  }
+
+  orgCurrentConcurrencyKeyFromQueue(queue: string) {
+    const orgId = this.normalizeQueue(queue).split(":")[1];
+
+    return `${constants.ORG_PART}:${orgId}:${constants.CURRENT_CONCURRENCY_PART}`;
+  }
+
+  envConcurrencyLimitKeyFromQueue(queue: string) {
+    const envId = this.normalizeQueue(queue).split(":")[3];
+
+    return `${constants.ENV_PART}:${envId}:${constants.CONCURRENCY_LIMIT_PART}`;
+  }
+
+  envCurrentConcurrencyKeyFromQueue(queue: string) {
+    const envId = this.normalizeQueue(queue).split(":")[3];
+
+    return `${constants.ENV_PART}:${envId}:${constants.CURRENT_CONCURRENCY_PART}`;
+  }
+
+  messageKey(messageId: string) {
+    return `${constants.MESSAGE_PART}:${messageId}`;
+  }
+
+  private shortId(id: string) {
+    // Return the last 12 characters of the id
+    return id.slice(-12);
+  }
+
+  private envKeySection(envId: string) {
+    return `${constants.ENV_PART}:${this.shortId(envId)}`;
+  }
+
+  private orgKeySection(orgId: string) {
+    return `${constants.ORG_PART}:${this.shortId(orgId)}`;
+  }
+
+  private queueSection(queue: string) {
+    return `${constants.QUEUE_PART}:${queue}`;
+  }
+
+  private concurrencyKeySection(concurrencyKey: string) {
+    return `${constants.CONCURRENCY_KEY_PART}:${concurrencyKey}`;
+  }
+
+  // This removes the leading prefix from the queue name if it exists
+  private normalizeQueue(queue: string) {
+    if (queue.startsWith(this._prefix)) {
+      return queue.slice(this._prefix.length);
+    }
+
+    return queue;
+  }
+}

apps/webapp/app/v3/marqs/priorityStrategy.server.ts

@@ -0,0 +1,150 @@
+import { RedisOptions } from "ioredis";
+import { MarQSQueuePriorityStrategy, PriorityStrategyChoice, QueueWithScores } from "./types";
+import { nanoid } from "nanoid";
+import seedrandom from "seedrandom";
+
+export type DynamicWeightedChoiceStrategyOptions = {
+  initialQueueSelectionSize: number;
+  redis: RedisOptions;
+};
+
+// This implementation of the priority strategy will "react" over time, giving more weight to queues that have been selected less frequently.
+// It will also change the next candidate selection range based on if previous choices only had queues that were at capacity.
+// Some other ideas:
+// - Implement a "cooldown" period for queues that have been selected recently
+// - Implement a "decay" for queues that have been selected recently
+//
+// The "memory" of this strategy is stored in Redis, to coordinate between multiple instances of the webapp (coming soon?)
+export class DynamicWeightedChoiceStrategy implements MarQSQueuePriorityStrategy {
+  constructor(private options: DynamicWeightedChoiceStrategyOptions) {}
+
+  chooseQueue(
+    queues: QueueWithScores[],
+    parentQueue: string,
+    selectionId: string
+  ): PriorityStrategyChoice {
+    throw new Error("Method not implemented.");
+  }
+
+  nextCandidateSelection(
+    parentQueue: string
+  ): Promise<{ range: [number, number]; selectionId: string }> {
+    throw new Error("Method not implemented.");
+  }
+}
+
+export type SimpleWeightedChoiceStrategyOptions = {
+  queueSelectionCount: number;
+  randomSeed?: string;
+};
+
+export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy {
+  private _nextRangesByParentQueue: Map<string, [number, number]> = new Map();
+  private _randomGenerator = seedrandom(this.options.randomSeed);
+
+  constructor(private options: SimpleWeightedChoiceStrategyOptions) {}
+
+  private nextRangeForParentQueue(parentQueue: string) {
+    return this._nextRangesByParentQueue.get(parentQueue) ?? [0, this.options.queueSelectionCount];
+  }
+
+  chooseQueue(
+    queues: QueueWithScores[],
+    parentQueue: string,
+    selectionId: string
+  ): PriorityStrategyChoice {
+    const filteredQueues = filterQueuesAtCapacity(queues);
+
+    if (filteredQueues.length === 0) {
+      if (queues.length === this.options.queueSelectionCount) {
+        const nextRangeForParentQueue = this.nextRangeForParentQueue(parentQueue);
+        const nextRange: [number, number] = nextRangeForParentQueue
+          ? [
+              nextRangeForParentQueue[1],
+              nextRangeForParentQueue[1] + this.options.queueSelectionCount,
+            ]
+          : [this.options.queueSelectionCount, this.options.queueSelectionCount * 2];
+        // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
+        this._nextRangesByParentQueue.set(parentQueue, nextRange);
+      } else {
+        this._nextRangesByParentQueue.delete(parentQueue);
+      }
+
+      return { abort: true };
+    }
+
+    this._nextRangesByParentQueue.delete(parentQueue);
+
+    const queueWeights = this.#calculateQueueWeights(filteredQueues);
+
+    return weightedRandomChoice(queueWeights, this._randomGenerator());
+  }
+
+  async nextCandidateSelection(
+    parentQueue: string
+  ): Promise<{ range: [number, number]; selectionId: string }> {
+    return { range: this.nextRangeForParentQueue(parentQueue), selectionId: nanoid(24) };
+  }
+
+  // This function calculates the weight of each queue based on the age of the queue and the capacity of the queue, env, and org
+  // First, it normalizes the age, queue capacity, env capacity, and org capacity to a value between 0 and 1 based on the maximum value of each
+  // Then, it calculates the weight of each queue based on the following factors:
+  // - Age is 50% of the weight
+  // - Queue capacity is 30% of the weight
+  // - Env capacity is 10% of the weight
+  // - Org capacity is 10% of the weight
+  #calculateQueueWeights(queues: QueueWithScores[]) {
+    const maximumAge = Math.max(...queues.map((queue) => queue.age));
+    const maximumQueueCapacity = Math.max(
+      ...queues.map((queue) => queue.capacities.queue.limit - queue.capacities.queue.current)
+    );
+    const maximumEnvCapacity = Math.max(
+      ...queues.map((queue) => queue.capacities.env.limit - queue.capacities.env.current)
+    );
+    const maximumOrgCapacity = Math.max(
+      ...queues.map((queue) => queue.capacities.org.limit - queue.capacities.org.current)
+    );
+
+    return queues.map(({ capacities, age, queue }) => {
+      const ageWeight = 0.5 * (age / maximumAge);
+      const queueWeight =
+        0.3 * (1 - (capacities.queue.limit - capacities.queue.current) / maximumQueueCapacity);
+      const envWeight =
+        0.1 * (1 - (capacities.env.limit - capacities.env.current) / maximumEnvCapacity);
+      const orgWeight =
+        0.1 * (1 - (capacities.org.limit - capacities.org.current) / maximumOrgCapacity);
+
+      return {
+        queue,
+        weight: ageWeight + queueWeight + envWeight + orgWeight,
+      };
+    });
+  }
+}
+
+function filterQueuesAtCapacity(queues: QueueWithScores[]) {
+  return queues.filter(
+    (queue) =>
+      queue.capacities.queue.current < queue.capacities.queue.limit &&
+      queue.capacities.env.current < queue.capacities.env.limit &&
+      queue.capacities.org.current < queue.capacities.org.limit
+  );
+}
+
+function weightedRandomChoice(
+  queues: Array<{ queue: string; weight: number }>,
+  randomNumber: number
+) {
+  const totalWeight = queues.reduce((acc, queue) => acc + queue.weight, 0);
+  const randomNum = randomNumber * totalWeight;
+  let weightSum = 0;
+
+  for (const queue of queues) {
+    weightSum += queue.weight;
+    if (randomNum <= weightSum) {
+      return queue.queue;
+    }
+  }
+
+  return queues[queues.length - 1].queue;
+}

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -1,5 +1,14 @@
-import { Context, ROOT_CONTEXT, Span, SpanKind, context, trace } from "@opentelemetry/api";
 import {
+  Context,
+  ROOT_CONTEXT,
+  Span,
+  SpanKind,
+  context,
+  propagation,
+  trace,
+} from "@opentelemetry/api";
+import {
+  Machine,
   ProdTaskRunExecution,
   ProdTaskRunExecutionPayload,
   TaskRunError,
@@ -20,7 +29,7 @@ import { z } from "zod";
 import { prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
 import { CancelAttemptService } from "../services/cancelAttempt.server";
 import { socketIo } from "../handleSocketIo.server";
@@ -30,19 +39,24 @@ import { findCurrentWorkerDeployment } from "../models/workerDeployment.server";
 
 const tracer = trace.getTracer("sharedQueueConsumer");
 
+const WithTraceContext = z.object({
+  traceparent: z.string().optional(),
+  tracestate: z.string().optional(),
+});
+
 const MessageBody = z.discriminatedUnion("type", [
-  z.object({
+  WithTraceContext.extend({
     type: z.literal("EXECUTE"),
     taskIdentifier: z.string(),
     checkpointEventId: z.string().optional(),
   }),
-  z.object({
+  WithTraceContext.extend({
     type: z.literal("RESUME"),
     completedAttemptIds: z.string().array(),
     resumableAttemptId: z.string(),
     checkpointEventId: z.string().optional(),
   }),
-  z.object({
+  WithTraceContext.extend({
     type: z.literal("RESUME_AFTER_DURATION"),
     resumableAttemptId: z.string(),
     checkpointEventId: z.string(),
@@ -56,6 +70,7 @@ export type SharedQueueConsumerOptions = {
   traceTimeoutSeconds?: number;
   nextTickInterval?: number;
   interval?: number;
+  parentContext?: Context;
 };
 
 export class SharedQueueConsumer {
@@ -82,6 +97,7 @@ export class SharedQueueConsumer {
       traceTimeoutSeconds: options.traceTimeoutSeconds ?? 60, // 60 seconds
       nextTickInterval: options.nextTickInterval ?? 1000, // 1 second
       interval: options.interval ?? 100, // 100ms
+      parentContext: options.parentContext ?? ROOT_CONTEXT,
     };
   }
 
@@ -186,8 +202,17 @@ export class SharedQueueConsumer {
     this.#doWork().finally(() => {});
   }
 
+  #endCurrentSpan() {
+    if (this._currentSpan) {
+      this._currentSpan.setAttribute("tasks.period.failures", this._taskFailures);
+      this._currentSpan.setAttribute("tasks.period.successes", this._taskSuccesses);
+      this._currentSpan.end();
+    }
+  }
+
   async #doWork() {
     if (!this._enabled) {
+      this.#endCurrentSpan();
       return;
     }
 
@@ -198,12 +223,9 @@ export class SharedQueueConsumer {
       this._currentSpanContext === undefined ||
       this._endSpanInNextIteration
     ) {
-      if (this._currentSpan) {
-        this._currentSpan.setAttribute("tasks.period.failures", this._taskFailures);
-        this._currentSpan.setAttribute("tasks.period.successes", this._taskSuccesses);
+      this.#endCurrentSpan();
 
-        this._currentSpan.end();
-      }
+      const parentContext = this._options.parentContext ?? ROOT_CONTEXT;
 
       // Create a new trace
       this._currentSpan = tracer.startSpan(
@@ -211,11 +233,11 @@ export class SharedQueueConsumer {
         {
           kind: SpanKind.CONSUMER,
         },
-        ROOT_CONTEXT
+        parentContext
       );
 
       // Get the span trace context
-      this._currentSpanContext = trace.setSpan(ROOT_CONTEXT, this._currentSpan);
+      this._currentSpanContext = trace.setSpan(parentContext, this._currentSpan);
 
       this._perTraceCountdown = this._options.maximumItemsPerTrace;
       this._lastNewTrace = new Date();
@@ -250,35 +272,12 @@ export class SharedQueueConsumer {
 
     logger.log("dequeueMessageInSharedQueue()", { queueMessage: message });
 
-    const envId = this.#envIdFromQueue(message.queue);
-
-    const environment = await prisma.runtimeEnvironment.findUnique({
-      include: {
-        organization: true,
-        project: true,
-      },
-      where: {
-        id: envId,
-      },
-    });
-
-    if (!environment) {
-      logger.error("Environment not found", {
-        queueMessage: message.data,
-        envId,
-      });
-
-      this.#ackAndDoMoreWork(message.messageId);
-      return;
-    }
-
     const messageBody = MessageBody.safeParse(message.data);
 
     if (!messageBody.success) {
       logger.error("Failed to parse message", {
         queueMessage: message.data,
         error: messageBody.error,
-        env: environment,
       });
 
       this.#ackAndDoMoreWork(message.messageId);
@@ -380,6 +379,7 @@ export class SharedQueueConsumer {
             lockedById: backgroundTask.id,
           },
           include: {
+            runtimeEnvironment: true,
             attempts: {
               take: 1,
               orderBy: { number: "desc" },
@@ -410,7 +410,7 @@ export class SharedQueueConsumer {
         const queue = await prisma.taskQueue.findUnique({
           where: {
             runtimeEnvironmentId_name: {
-              runtimeEnvironmentId: environment.id,
+              runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId,
               name: lockedTaskRun.queue,
             },
           },
@@ -436,12 +436,29 @@ export class SharedQueueConsumer {
             backgroundWorkerTaskId: backgroundTask.id,
             status: "PENDING" as const,
             queueId: queue.id,
-            runtimeEnvironmentId: environment.id,
+            runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId,
+          },
+          include: {
+            backgroundWorkerTask: true,
           },
         });
 
         const isRetry = taskRunAttempt.number > 1;
 
+        const { machineConfig } = taskRunAttempt.backgroundWorkerTask;
+        const machine = Machine.safeParse(machineConfig ?? {});
+
+        if (!machine.success) {
+          logger.error("Failed to parse machine config", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+            attemptId: taskRunAttempt.id,
+            machineConfig,
+          });
+
+          await this.#ackAndDoMoreWork(message.messageId);
+          return;
+        }
         try {
           if (messageBody.data.checkpointEventId) {
             const restoreService = new RestoreCheckpointService();
@@ -470,11 +487,16 @@ export class SharedQueueConsumer {
               backgroundWorkerId: deployment.worker.friendlyId,
               data: {
                 type: "SCHEDULE_ATTEMPT",
-                id: taskRunAttempt.id,
                 image: deployment.imageReference,
-                envId: environment.id,
-                runId: taskRunAttempt.taskRunId,
                 version: deployment.version,
+                machine: machine.data,
+                // identifiers
+                id: taskRunAttempt.id,
+                envId: lockedTaskRun.runtimeEnvironment.id,
+                envType: lockedTaskRun.runtimeEnvironment.type,
+                orgId: lockedTaskRun.runtimeEnvironment.organizationId,
+                projectId: lockedTaskRun.runtimeEnvironment.projectId,
+                runId: taskRunAttempt.taskRunId,
               },
             });
           }
@@ -602,7 +624,7 @@ export class SharedQueueConsumer {
         const queue = await prisma.taskQueue.findUnique({
           where: {
             runtimeEnvironmentId_name: {
-              runtimeEnvironmentId: environment.id,
+              runtimeEnvironmentId: resumableAttempt.runtimeEnvironmentId,
               name: resumableRun.queue,
             },
           },
@@ -731,10 +753,6 @@ export class SharedQueueConsumer {
     return;
   }
 
-  #envIdFromQueue(queueName: string) {
-    return queueName.split(":")[1];
-  }
-
   #doMoreWork(intervalInMs = this._options.interval) {
     setTimeout(() => this.#doWork(), intervalInMs);
   }

apps/webapp/app/v3/marqs/types.ts

@@ -0,0 +1,79 @@
+import { RedisOptions } from "ioredis";
+import { z } from "zod";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+
+export type QueueCapacity = {
+  current: number;
+  limit: number;
+};
+
+export type QueueCapacities = {
+  queue: QueueCapacity;
+  env: QueueCapacity;
+  org: QueueCapacity;
+};
+
+export type QueueWithScores = {
+  queue: string;
+  capacities: QueueCapacities;
+  age: number;
+};
+
+export interface MarQSKeyProducer {
+  queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string;
+  envConcurrencyLimitKey(env: AuthenticatedEnvironment): string;
+  orgConcurrencyLimitKey(env: AuthenticatedEnvironment): string;
+  queueKey(env: AuthenticatedEnvironment, queue: string, concurrencyKey?: string): string;
+  envSharedQueueKey(env: AuthenticatedEnvironment): string;
+  concurrencyLimitKeyFromQueue(queue: string): string;
+  currentConcurrencyKeyFromQueue(queue: string): string;
+  orgConcurrencyLimitKeyFromQueue(queue: string): string;
+  orgCurrentConcurrencyKeyFromQueue(queue: string): string;
+  envConcurrencyLimitKeyFromQueue(queue: string): string;
+  envCurrentConcurrencyKeyFromQueue(queue: string): string;
+  messageKey(messageId: string): string;
+}
+
+export type PriorityStrategyChoice = string | { abort: true };
+
+export interface MarQSQueuePriorityStrategy {
+  /**
+   * chooseQueue is called to select the next queue to process a message from
+   *
+   * @param queues
+   * @param parentQueue
+   * @param selectionId
+   *
+   * @returns The queue to process the message from, or an object with `abort: true` if no queue is available
+   */
+  chooseQueue(
+    queues: Array<QueueWithScores>,
+    parentQueue: string,
+    selectionId: string
+  ): PriorityStrategyChoice;
+
+  /**
+   * This function is called to get the next candidate selection for the queue
+   * The `range` is used to select the set of queues that will be considered for the next selection (passed to chooseQueue)
+   * The `selectionId` is used to identify the selection and should be passed to chooseQueue
+   *
+   * @param parentQueue The parent queue that holds the candidate queues
+   *
+   * @returns The scores and the selectionId for the next candidate selection
+   */
+  nextCandidateSelection(
+    parentQueue: string
+  ): Promise<{ range: [number, number]; selectionId: string }>;
+}
+
+export const MessagePayload = z.object({
+  version: z.literal("1"),
+  data: z.record(z.unknown()),
+  queue: z.string(),
+  messageId: z.string(),
+  timestamp: z.number(),
+  parentQueue: z.string(),
+  concurrencyKey: z.string().optional(),
+});
+
+export type MessagePayload = z.infer<typeof MessagePayload>;

apps/webapp/app/v3/registryProxy.server.ts

@@ -259,6 +259,18 @@ export class RegistryProxy {
         proxyRes.pipe(response, { end: true });
       });
 
+      request.on("close", () => {
+        logger.debug("Client closed the connection");
+        proxyReq.destroy();
+        cleanupTempFile();
+      });
+
+      request.on("abort", () => {
+        logger.debug("Client aborted the connection");
+        proxyReq.destroy(); // Abort the proxied request
+        cleanupTempFile(); // Clean up the temporary file if necessary
+      });
+
       if (tempFilePath) {
         const readStream = createReadStream(tempFilePath);
 
@@ -427,14 +439,22 @@ function initializeProxy() {
   });
 }
 
-async function streamRequestBodyToTempFile(request: IncomingMessage): Promise<string> {
-  const tempDir = await mkdtemp(`${tmpdir()}/`);
-  const tempFilePath = `${tempDir}/requestBody.tmp`;
-  const writeStream = createWriteStream(tempFilePath);
+async function streamRequestBodyToTempFile(request: IncomingMessage): Promise<string | undefined> {
+  try {
+    const tempDir = await mkdtemp(`${tmpdir()}/`);
+    const tempFilePath = `${tempDir}/requestBody.tmp`;
+    const writeStream = createWriteStream(tempFilePath);
 
-  await pipeline(request, writeStream);
+    await pipeline(request, writeStream);
 
-  return tempFilePath;
+    return tempFilePath;
+  } catch (error) {
+    logger.error("Failed to stream request body to temp file", {
+      error: error instanceof Error ? error.message : error,
+    });
+
+    return;
+  }
 }
 
 type DockerImageParts = {

apps/webapp/app/v3/services/cancelAttempt.server.ts

@@ -1,6 +1,6 @@
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { eventRepository } from "../eventRepository.server";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -1,6 +1,6 @@
 import { TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 import { eventRepository } from "../eventRepository.server";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { devPubSub } from "../marqs/devPubSub.server";
 import { BaseService } from "./baseService.server";
 import { socketIo } from "../handleSocketIo.server";

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -12,7 +12,7 @@ import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { safeJsonParse } from "~/utils/json";
 import { eventRepository } from "../eventRepository.server";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { CancelAttemptService } from "./cancelAttempt.server";
 import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
@@ -105,7 +105,7 @@ export class CompleteAttemptService extends BaseService {
       attributes: {
         isError: false,
         output:
-          completion.outputType === "application/store"
+          completion.outputType === "application/store" || completion.outputType === "text/plain"
             ? completion.output
             : completion.output
             ? (safeJsonParse(completion.output) as Attributes)

apps/webapp/app/v3/services/createBackgroundWorker.server.ts

@@ -4,10 +4,11 @@ import { Prisma, PrismaClientOrTransaction } from "~/db.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { marqs } from "../marqs.server";
+import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
 import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion";
 import { BaseService } from "./baseService.server";
 import { projectPubSub } from "./projectPubSub.server";
+import { env } from "~/env.server";
 
 export class CreateBackgroundWorkerService extends BaseService {
   public async call(
@@ -68,14 +69,39 @@ export class CreateBackgroundWorkerService extends BaseService {
 
       await createBackgroundTasks(body.metadata.tasks, backgroundWorker, environment, this._prisma);
 
-      //send a notification that a new worker has been created
-      await projectPubSub.publish(`project:${project.id}:env:${environment.id}`, "WORKER_CREATED", {
-        environmentId: environment.id,
-        environmentType: environment.type,
-        createdAt: backgroundWorker.createdAt,
-        taskCount: body.metadata.tasks.length,
-        type: "local",
-      });
+      try {
+        //send a notification that a new worker has been created
+        await projectPubSub.publish(
+          `project:${project.id}:env:${environment.id}`,
+          "WORKER_CREATED",
+          {
+            environmentId: environment.id,
+            environmentType: environment.type,
+            createdAt: backgroundWorker.createdAt,
+            taskCount: body.metadata.tasks.length,
+            type: "local",
+          }
+        );
+
+        await marqs?.updateEnvConcurrencyLimits(environment);
+      } catch (err) {
+        logger.error(
+          "Error publishing WORKER_CREATED event or updating global concurrency limits",
+          {
+            error:
+              err instanceof Error
+                ? {
+                    name: err.name,
+                    message: err.message,
+                    stack: err.stack,
+                  }
+                : err,
+            project,
+            environment,
+            backgroundWorker,
+          }
+        );
+      }
 
       return backgroundWorker;
     });
@@ -85,7 +111,7 @@ export class CreateBackgroundWorkerService extends BaseService {
 export async function createBackgroundTasks(
   tasks: TaskResource[],
   worker: BackgroundWorker,
-  env: AuthenticatedEnvironment,
+  environment: AuthenticatedEnvironment,
   prisma: PrismaClientOrTransaction
 ) {
   for (const task of tasks) {
@@ -101,10 +127,28 @@ export async function createBackgroundTasks(
           exportName: task.exportName,
           retryConfig: task.retry,
           queueConfig: task.queue,
+          machineConfig: task.machine,
         },
       });
 
-      const queueName = task.queue?.name ?? `task/${task.id}`;
+      let queueName = sanitizeQueueName(task.queue?.name ?? `task/${task.id}`);
+
+      // Check that the queuename is not an empty string
+      if (!queueName) {
+        queueName = sanitizeQueueName(`task/${task.id}`);
+      }
+
+      const concurrencyLimit =
+        typeof task.queue?.concurrencyLimit === "number"
+          ? Math.max(
+              Math.min(
+                task.queue.concurrencyLimit,
+                environment.maximumConcurrencyLimit,
+                environment.organization.maximumConcurrencyLimit
+              ),
+              0
+            )
+          : null;
 
       const taskQueue = await prisma.taskQueue.upsert({
         where: {
@@ -114,13 +158,13 @@ export async function createBackgroundTasks(
           },
         },
         update: {
-          concurrencyLimit: task.queue?.concurrencyLimit,
+          concurrencyLimit,
           rateLimit: task.queue?.rateLimit,
         },
         create: {
           friendlyId: generateFriendlyId("queue"),
           name: queueName,
-          concurrencyLimit: task.queue?.concurrencyLimit,
+          concurrencyLimit,
           runtimeEnvironmentId: worker.runtimeEnvironmentId,
           projectId: worker.projectId,
           rateLimit: task.queue?.rateLimit,
@@ -129,7 +173,11 @@ export async function createBackgroundTasks(
       });
 
       if (taskQueue.concurrencyLimit) {
-        await marqs?.updateQueueConcurrency(env, taskQueue.name, taskQueue.concurrencyLimit);
+        await marqs?.updateQueueConcurrencyLimits(
+          environment,
+          taskQueue.name,
+          taskQueue.concurrencyLimit
+        );
       }
     } catch (error) {
       if (error instanceof Prisma.PrismaClientKnownRequestError) {

apps/webapp/app/v3/services/createCheckpoint.server.ts

@@ -6,7 +6,7 @@ import type {
 } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
 import { BaseService } from "./baseService.server";
 

apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts

@@ -6,6 +6,8 @@ import { BaseService } from "./baseService.server";
 import { createBackgroundTasks } from "./createBackgroundWorker.server";
 import { CURRENT_DEPLOYMENT_LABEL } from "~/consts";
 import { projectPubSub } from "./projectPubSub.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { logger } from "~/services/logger.server";
 
 export class CreateDeployedBackgroundWorkerService extends BaseService {
   public async call(
@@ -76,18 +78,23 @@ export class CreateDeployedBackgroundWorkerService extends BaseService {
         },
       });
 
-      //send a notification that a new worker has been created
-      await projectPubSub.publish(
-        `project:${environment.projectId}:env:${environment.id}`,
-        "WORKER_CREATED",
-        {
-          environmentId: environment.id,
-          environmentType: environment.type,
-          createdAt: backgroundWorker.createdAt,
-          taskCount: body.metadata.tasks.length,
-          type: "deployed",
-        }
-      );
+      try {
+        //send a notification that a new worker has been created
+        await projectPubSub.publish(
+          `project:${environment.projectId}:env:${environment.id}`,
+          "WORKER_CREATED",
+          {
+            environmentId: environment.id,
+            environmentType: environment.type,
+            createdAt: backgroundWorker.createdAt,
+            taskCount: body.metadata.tasks.length,
+            type: "deployed",
+          }
+        );
+        await marqs?.updateEnvConcurrencyLimits(environment);
+      } catch (err) {
+        logger.error("Failed to publish WORKER_CREATED event", { err });
+      }
 
       return backgroundWorker;
     });

apps/webapp/app/v3/services/indexDeployment.server.ts

@@ -49,9 +49,13 @@ export class IndexDeploymentService extends BaseService {
         version: "v1",
         shortCode: deployment.shortCode,
         imageTag: deployment.imageReference,
-        envId: deployment.environmentId,
         apiKey: deployment.environment.apiKey,
         apiUrl: env.APP_ORIGIN,
+        // identifiers
+        envId: deployment.environmentId,
+        envType: deployment.environment.type,
+        projectId: deployment.projectId,
+        orgId: deployment.environment.organizationId,
       });
 
       logger.debug("Index ACK received", { responses });

apps/webapp/app/v3/services/restoreCheckpoint.server.ts

@@ -3,6 +3,7 @@ import { logger } from "~/services/logger.server";
 import { socketIo } from "../handleSocketIo.server";
 import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
 import { BaseService } from "./baseService.server";
+import { Machine } from "@trigger.dev/core/v3";
 
 const RESTORABLE_RUN_STATUSES: TaskRunStatus[] = ["WAITING_TO_RESUME"];
 const RESTORABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["PAUSED"];
@@ -30,8 +31,14 @@ export class RestoreCheckpointService extends BaseService {
             attempt: {
               select: {
                 status: true,
+                backgroundWorkerTask: {
+                  select: {
+                    machineConfig: true,
+                  },
+                },
               },
             },
+            runtimeEnvironment: true,
           },
         },
       },
@@ -63,17 +70,34 @@ export class RestoreCheckpointService extends BaseService {
       return;
     }
 
+    const { machineConfig } = checkpoint.attempt.backgroundWorkerTask;
+    const machine = Machine.safeParse(machineConfig ?? {});
+
+    if (!machine.success) {
+      logger.error("Failed to parse machine config", {
+        attemptId: checkpoint.attemptId,
+        machineConfig: checkpoint.attempt.backgroundWorkerTask.machineConfig,
+      });
+      return;
+    }
+
     const eventService = new CreateCheckpointRestoreEventService(this._prisma);
     await eventService.restore({ checkpointId: checkpoint.id });
 
     socketIo.providerNamespace.emit("RESTORE", {
       version: "v1",
-      checkpointId: checkpoint.id,
-      runId: checkpoint.runId,
       type: checkpoint.type,
       location: checkpoint.location,
       reason: checkpoint.reason ?? undefined,
       imageRef: checkpoint.imageRef,
+      machine: machine.data,
+      // identifiers
+      checkpointId: checkpoint.id,
+      envId: checkpoint.runtimeEnvironment.id,
+      envType: checkpoint.runtimeEnvironment.type,
+      orgId: checkpoint.runtimeEnvironment.organizationId,
+      projectId: checkpoint.runtimeEnvironment.projectId,
+      runId: checkpoint.runId,
     });
 
     return checkpoint;

apps/webapp/app/v3/services/resumeAttempt.server.ts

@@ -6,7 +6,7 @@ import {
 } from "@trigger.dev/core/v3";
 import { $transaction } from "~/db.server";
 import { logger } from "~/services/logger.server";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { socketIo } from "../handleSocketIo.server";
 import { sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
 import { BaseService } from "./baseService.server";

apps/webapp/app/v3/services/resumeBatchRun.server.ts

@@ -1,6 +1,6 @@
 import { PrismaClientOrTransaction } from "~/db.server";
 import { workerQueue } from "~/services/worker.server";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 

apps/webapp/app/v3/services/resumeTaskDependency.server.ts

@@ -1,6 +1,6 @@
 import { PrismaClientOrTransaction } from "~/db.server";
 import { workerQueue } from "~/services/worker.server";
-import { marqs } from "../marqs.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -1,5 +1,4 @@
 import {
-  PRIMARY_VARIANT,
   SemanticInternalAttributes,
   TriggerTaskRequestBody,
   packetRequiresOffloading,
@@ -10,10 +9,9 @@ import { $transaction } from "~/db.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { eventRepository } from "../eventRepository.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { marqs } from "../marqs.server";
-import { BaseService } from "./baseService.server";
+import { marqs } from "~/v3/marqs/index.server";
 import { uploadToObjectStore } from "../r2.server";
-import { logger } from "~/services/logger.server";
+import { BaseService } from "./baseService.server";
 
 export type TriggerTaskServiceOptions = {
   idempotencyKey?: string;

apps/webapp/app/v3/sharedSocketConnection.ts

@@ -10,7 +10,56 @@ import { Evt } from "evt";
 import { randomUUID } from "node:crypto";
 import { logger } from "~/services/logger.server";
 import { SharedQueueConsumer } from "./marqs/sharedQueueConsumer.server";
-import { DisconnectReason, Namespace, Socket } from "socket.io";
+import type { DisconnectReason, Namespace, Socket } from "socket.io";
+import { ROOT_CONTEXT, Span, SpanKind, trace } from "@opentelemetry/api";
+import { env } from "~/env.server";
+
+const tracer = trace.getTracer("sharedQueueConsumerPool");
+
+interface SharedQueueConsumerPoolOptions {
+  sender: ZodMessageSender<typeof serverWebsocketMessages>;
+  poolSize: number;
+}
+
+class SharedQueueConsumerPool {
+  #consumers: SharedQueueConsumer[];
+  #span: Span;
+
+  constructor(opts: SharedQueueConsumerPoolOptions) {
+    this.#span = tracer.startSpan(
+      "SharedQueueConsumerPool()",
+      {
+        kind: SpanKind.CONSUMER,
+        attributes: {
+          "pool.size": opts.poolSize,
+        },
+      },
+      ROOT_CONTEXT
+    );
+
+    const spanContext = trace.setSpan(ROOT_CONTEXT, this.#span);
+
+    this.#consumers = Array(opts.poolSize)
+      .fill(null)
+      .map(
+        () =>
+          new SharedQueueConsumer(opts.sender, {
+            interval: env.SHARED_QUEUE_CONSUMER_INTERVAL_MS,
+            nextTickInterval: env.SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS,
+            parentContext: spanContext,
+          })
+      );
+  }
+
+  async start() {
+    await Promise.allSettled(this.#consumers.map((consumer) => consumer.start()));
+  }
+
+  async stop() {
+    await Promise.allSettled(this.#consumers.map((consumer) => consumer.stop()));
+    this.#span.end();
+  }
+}
 
 interface SharedSocketConnectionOptions {
   namespace: Namespace<
@@ -22,6 +71,7 @@ interface SharedSocketConnectionOptions {
     MessageCatalogToSocketIoEvents<typeof serverWebsocketMessages>
   >;
   logger?: StructuredLogger;
+  poolSize?: number;
 }
 
 export class SharedSocketConnection {
@@ -29,8 +79,9 @@ export class SharedSocketConnection {
   public onClose: Evt<DisconnectReason> = new Evt();
 
   private _sender: ZodMessageSender<typeof serverWebsocketMessages>;
-  private _sharedConsumer: SharedQueueConsumer;
+  private _sharedQueueConsumerPool: SharedQueueConsumerPool;
   private _messageHandler: ZodMessageHandler<typeof clientWebsocketMessages>;
+  private _defaultPoolSize = 10;
 
   constructor(opts: SharedSocketConnectionOptions) {
     this.id = randomUUID();
@@ -50,9 +101,13 @@ export class SharedSocketConnection {
       },
     });
 
-    this._sharedConsumer = new SharedQueueConsumer(this._sender, {
-      interval: 100,
-      nextTickInterval: 1000,
+    logger.log("Starting SharedQueueConsumer pool", {
+      poolSize: opts.poolSize ?? this._defaultPoolSize,
+    });
+
+    this._sharedQueueConsumerPool = new SharedQueueConsumerPool({
+      poolSize: opts.poolSize ?? this._defaultPoolSize,
+      sender: this._sender,
     });
 
     opts.socket.on("disconnect", this.#handleClose.bind(this));
@@ -62,7 +117,7 @@ export class SharedSocketConnection {
       schema: clientWebsocketMessages,
       messages: {
         READY_FOR_TASKS: async (payload) => {
-          this._sharedConsumer.start();
+          this._sharedQueueConsumerPool.start();
         },
         BACKGROUND_WORKER_DEPRECATED: async (payload) => {
           // await this._sharedConsumer.deprecateBackgroundWorker(payload.backgroundWorkerId);
@@ -89,7 +144,7 @@ export class SharedSocketConnection {
   }
 
   async #handleClose(ev: DisconnectReason) {
-    await this._sharedConsumer.stop();
+    await this._sharedQueueConsumerPool.stop();
 
     this.onClose.post(ev);
   }

apps/webapp/app/v3/tracer.server.ts

@@ -9,7 +9,7 @@ import {
   trace,
 } from "@opentelemetry/api";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-import { registerInstrumentations } from "@opentelemetry/instrumentation";
+import { InstrumentationOption, registerInstrumentations } from "@opentelemetry/instrumentation";
 import { ExpressInstrumentation } from "@opentelemetry/instrumentation-express";
 import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
 import { Resource } from "@opentelemetry/resources";
@@ -23,12 +23,13 @@ import {
   TraceIdRatioBasedSampler,
 } from "@opentelemetry/sdk-trace-base";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
-import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
+import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
 import { PrismaInstrumentation } from "@prisma/instrumentation";
 import { env } from "~/env.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { singleton } from "~/utils/singleton";
 import { LoggerSpanExporter } from "./telemetry/loggerExporter.server";
+
 class CustomWebappSampler implements Sampler {
   constructor(private readonly _baseSampler: Sampler) {}
 
@@ -44,10 +45,7 @@ class CustomWebappSampler implements Sampler {
     const parentContext = trace.getSpanContext(context);
 
     // Exclude Prisma spans (adjust this logic as needed for your use case)
-    if (
-      !parentContext &&
-      ((attributes && attributes["model"] && attributes["method"]) || name.includes("prisma"))
-    ) {
+    if (!parentContext && name.includes("prisma")) {
       return { decision: SamplingDecision.NOT_RECORD };
     }
 
@@ -65,29 +63,46 @@ export const tracer = singleton("tracer", getTracer);
 function getTracer() {
   diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
 
+  const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPING_RATE, 10), 1);
+
   const provider = new NodeTracerProvider({
     forceFlushTimeoutMillis: 500,
     resource: new Resource({
-      [SemanticResourceAttributes.SERVICE_NAME]: "trigger.dev",
+      [SEMRESATTRS_SERVICE_NAME]: env.SERVICE_NAME,
     }),
     sampler: new ParentBasedSampler({
-      root: new CustomWebappSampler(
-        new TraceIdRatioBasedSampler(env.APP_ENV === "development" ? 1.0 : 0.05)
-      ), // 5% sampling
+      root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)), // 5% sampling
     }), // 5% sampling
   });
 
-  if (env.OTLP_EXPORTER_TRACES_URL) {
+  if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {
     const exporter = new OTLPTraceExporter({
-      url: env.OTLP_EXPORTER_TRACES_URL,
+      url: env.INTERNAL_OTEL_TRACE_EXPORTER_URL,
       timeoutMillis: 1000,
+      headers:
+        env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME &&
+        env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE
+          ? {
+              [env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME]:
+                env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE,
+            }
+          : undefined,
     });
 
-    provider.addSpanProcessor(new BatchSpanProcessor(exporter));
+    provider.addSpanProcessor(
+      new BatchSpanProcessor(exporter, {
+        maxExportBatchSize: 512,
+        scheduledDelayMillis: 200,
+        exportTimeoutMillis: 30000,
+        maxQueueSize: 2048,
+      })
+    );
 
-    console.log(`⚡ Tracer: OTLP exporter enabled to ${env.OTLP_EXPORTER_TRACES_URL}`);
+    console.log(`🔦 Tracer: OTLP exporter enabled to ${env.INTERNAL_OTEL_TRACE_EXPORTER_URL}`);
   } else {
-    if (env.LOG_TELEMETRY === "true") {
+    if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") {
+      console.log(`🔦 Tracer: Logger exporter enabled`);
+
       const loggerExporter = new LoggerSpanExporter();
 
       provider.addSpanProcessor(new SimpleSpanProcessor(loggerExporter));
@@ -96,13 +111,18 @@ function getTracer() {
 
   provider.register();
 
+  let instrumentations: InstrumentationOption[] = [
+    new HttpInstrumentation(),
+    new ExpressInstrumentation(),
+  ];
+
+  if (env.INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED === "1") {
+    instrumentations.push(new PrismaInstrumentation());
+  }
+
   registerInstrumentations({
     tracerProvider: provider,
-    instrumentations: [
-      new HttpInstrumentation(),
-      new ExpressInstrumentation(),
-      new PrismaInstrumentation(),
-    ],
+    instrumentations,
   });
 
   return provider.getTracer("trigger.dev", "3.0.0.dp.1");

apps/webapp/package.json

@@ -19,7 +19,8 @@
     "generate:sourcemaps": "remix build --sourcemap",
     "clean:sourcemaps": "run-s clean:sourcemaps:*",
     "clean:sourcemaps:public": "rimraf ./build/**/*.map",
-    "clean:sourcemaps:build": "rimraf ./public/build/**/*.map"
+    "clean:sourcemaps:build": "rimraf ./public/build/**/*.map",
+    "test": "vitest"
   },
   "eslintIgnore": [
     "/node_modules",
@@ -140,6 +141,7 @@
     "remix-auth-github": "^1.6.0",
     "remix-typedjson": "0.3.1",
     "remix-utils": "^7.1.0",
+    "seedrandom": "^3.0.5",
     "semver": "^7.5.0",
     "simple-oauth2": "^5.0.0",
     "simplur": "^3.0.1",
@@ -186,6 +188,7 @@
     "@types/react": "18.2.69",
     "@types/react-collapse": "^5.0.4",
     "@types/react-dom": "18.2.7",
+    "@types/seedrandom": "^3.0.8",
     "@types/semver": "^7.3.13",
     "@types/simple-oauth2": "^5.0.4",
     "@types/slug": "^5.0.3",
@@ -214,7 +217,8 @@
     "tailwindcss": "3.4.1",
     "ts-node": "^10.7.0",
     "tsconfig-paths": "^3.14.1",
-    "typescript": "^5.1.6"
+    "typescript": "^5.1.6",
+    "vitest": "^1.4.0"
   },
   "engines": {
     "node": ">=16.0.0"

apps/webapp/test/marqs.test.ts

@@ -0,0 +1,263 @@
+import { SimpleWeightedChoiceStrategy } from "../app/v3/marqs/priorityStrategy.server";
+
+describe("SimpleWeightedChoiceStrategy", () => {
+  it("should use a weighted random choice algorithm to choose a queue", async () => {
+    const stategy = new SimpleWeightedChoiceStrategy({
+      queueSelectionCount: 3,
+      randomSeed: "test",
+    });
+
+    const chosenQueue = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue3",
+          age: 12828,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue",
+      "selectionId"
+    );
+
+    expect(chosenQueue).toEqual("queue3");
+  });
+
+  it("should filter out queues if any capacity is full", async () => {
+    const stategy = new SimpleWeightedChoiceStrategy({
+      queueSelectionCount: 3,
+      randomSeed: "test",
+    });
+
+    const chosenQueue = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 10, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 10, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue3",
+          age: 12828,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 10, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue",
+      "selectionId"
+    );
+
+    expect(chosenQueue).toEqual({ abort: true });
+
+    const nextSelection = await stategy.nextCandidateSelection("parentQueue");
+
+    expect(nextSelection).toEqual({ range: [3, 6], selectionId: expect.any(String) });
+
+    // Now pass some queues that have some capacity
+    const chosenQueue2 = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue3",
+          age: 12828,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue",
+      "selectionId"
+    );
+
+    expect(chosenQueue2).toEqual("queue3");
+
+    const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");
+
+    expect(nextSelection2).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+  });
+
+  it("should adjust the next filter range only if passed the maximum number of queues", async () => {
+    const stategy = new SimpleWeightedChoiceStrategy({
+      queueSelectionCount: 3,
+      randomSeed: "test",
+    });
+
+    const chosenQueue = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 10, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 10, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue",
+      "selectionId"
+    );
+
+    expect(chosenQueue).toEqual({ abort: true });
+
+    const nextSelection = await stategy.nextCandidateSelection("parentQueue");
+
+    expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+  });
+
+  it("should adjust the next candidate range ONLY for the matching parent queue", async () => {
+    const stategy = new SimpleWeightedChoiceStrategy({
+      queueSelectionCount: 3,
+      randomSeed: "test",
+    });
+
+    const chosenQueue = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 10, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 10, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue3",
+          age: 12828,
+          capacities: {
+            queue: { current: 10, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue",
+      "selectionId"
+    );
+
+    expect(chosenQueue).toEqual({ abort: true });
+
+    const nextSelection = await stategy.nextCandidateSelection("parentQueue2");
+
+    expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+
+    const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");
+
+    expect(nextSelection2).toEqual({ range: [3, 6], selectionId: expect.any(String) });
+
+    const chosenQueue2 = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue3",
+          age: 12828,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue2",
+      "selectionId"
+    );
+
+    expect(chosenQueue2).toEqual("queue3");
+
+    const nextSelection3 = await stategy.nextCandidateSelection("parentQueue2");
+
+    expect(nextSelection3).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+  });
+});

apps/webapp/tsconfig.json

@@ -2,6 +2,7 @@
   "exclude": ["./cypress", "./cypress.config.ts"],
   "include": ["remix.env.d.ts", "**/*.ts", "**/*.tsx"],
   "compilerOptions": {
+    "types": ["vitest/globals"],
     "lib": ["DOM", "DOM.Iterable", "ES2019"],
     "isolatedModules": true,
     "esModuleInterop": true,

apps/webapp/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["test/**/*.test.ts"],
+    globals: true,
+  },
+});

docs/integrations/apis/shopify.mdx

@@ -63,7 +63,9 @@ You can use Personal Access Tokens to authenticate with Shopify and get started
 
 To create the tokens on Shopify, login and [follow the instructions](https://help.shopify.com/en/manual/apps/app-types/custom-apps#create-and-install-a-custom-app).
 
-The [required scopes](https://shopify.dev/docs/api/usage/access-scopes#authenticated-access-scopes) depend on the tasks you wish to perform and which webhooks you intend to receive. Webhooks will generally need read access to the respective Shopify resource.
+The [required scopes](https://shopify.dev/docs/api/usage/access-scopes#authenticated-access-scopes) depend on the tasks you wish to perform and which webhooks you intend to receive.
+
+Webhooks will generally need read access to the respective Shopify resource at the very least. If the topic you subscribe to is triggered by an action that requires write access, e.g. `orders/create`, then write access is also required, e.g. `write_orders`.
 
 Additionally, you will also have to provide your shop domain.
 

docs/mint.json

@@ -452,6 +452,7 @@
         "v3/trigger-folder",
         "v3/tasks-overview",
         "v3/triggering",
+        "v3/apikeys",
         {
           "group": "Task types",
           "pages": ["v3/tasks-regular", "v3/tasks-scheduled", "v3/tasks-zod", "v3/tasks-webhooks"]

docs/v3/apikeys.mdx

@@ -0,0 +1,12 @@
+---
+title: "API keys"
+description: "How to authenticate with Trigger.dev so you can trigger tasks."
+---
+
+### Authentication and your secret keys
+
+When you [trigger a task](/v3/triggering) from your backend code, you need to set the `TRIGGER_SECRET_KEY` environment variable.
+
+Each environment has its own secret key. You can find the value on the API keys page in the Trigger.dev dashboard:
+
+![How to find your secret key](/images/v3/api-keys.png)

docs/v3/deploy-environment-variables.mdx

@@ -18,7 +18,7 @@ We deploy your tasks and scale them up and down when they are triggered. So any
 
 <Step title="Add your environment variables">
   You can add values for your local dev environment, staging and prod. ![Environment variables
-  page](/images/v3/environment-variables-page.jpg){" "}
+  page](/images/v3/environment-variables-panel.jpg)
 </Step>
 
 </Steps>

docs/v3/triggering.mdx

@@ -23,6 +23,10 @@ You can trigger any task from your backend code, using either `trigger()` or `ba
   Trigger.dev API key to the world.
 </Note>
 
+### Authentication
+
+When you trigger a task from your backend code, you need to set the `TRIGGER_SECRET_KEY` environment variable. You can find the value on the API keys page in the Trigger.dev dashboard. [More info on API keys](/v3/apikeys).
+
 ### trigger()
 
 Triggers a single run of a task with the payload you pass in, and any options you specify. It does NOT wait for the result, you cannot do that from outside a task.

integrations/airtable/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/airtable
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/airtable/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/airtable",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "Trigger.dev integration for airtable",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "airtable": "^0.12.1",
     "zod": "3.22.3"
   },

integrations/github/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/github
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/github/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/github",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "The official GitHub integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -30,8 +30,8 @@
     "@octokit/request-error": "^5.0.1",
     "@octokit/webhooks": "^12.0.10",
     "octokit": "^3.1.2",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "zod": "3.22.3"
   },
   "engines": {

integrations/linear/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/linear
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/linear/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/linear",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "Trigger.dev integration for @linear/sdk",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@linear/sdk": "^8.0.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "zod": "3.22.3"
   },
   "engines": {

integrations/openai/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/slack
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/openai/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/openai",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "The official OpenAI integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -42,8 +42,8 @@
   },
   "dependencies": {
     "openai": "^4.16.1",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4"
   },
   "engines": {
     "node": ">=18.0.0"

integrations/plain/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/plain
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/plain/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/plain",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "The official Plain.com integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "build:tsup": "tsup"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "@team-plain/typescript-sdk": "^2.7.0"
   },
   "engines": {

integrations/replicate/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/replicate
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/replicate/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/replicate",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "Trigger.dev integration for replicate",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "replicate": "^0.18.1",
     "zod": "3.22.3"
   },

integrations/resend/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/resend
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/resend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/resend",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "The official Resend.com integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "build:tsup": "tsup"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "resend": "^2.1.0"
   },
   "engines": {

integrations/sendgrid/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/sendgrid
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/sendgrid/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/sendgrid",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "Trigger.dev integration for @sendgrid/mail",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@sendgrid/mail": "^7.7.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4"
   },
   "engines": {
     "node": ">=16.8.0"

integrations/shopify/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/shopify
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/shopify/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/shopify",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "Trigger.dev integration for @shopify/shopify-api",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@shopify/shopify-api": "^8.0.2",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
     "zod": "3.22.3"
   },
   "engines": {

integrations/slack/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @trigger.dev/slack
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/slack/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/slack",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "The official Slack integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@slack/web-api": "^6.8.1",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "zod": "3.22.3"
   },
   "engines": {

integrations/stripe/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/stripe
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/stripe/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/stripe",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "Trigger.dev integration for stripe",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "stripe": "^12.14.0",
     "zod": "3.22.3"
   },

integrations/supabase/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/supabase
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/supabase/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/supabase",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "Trigger.dev integration for @supabase/supabase-js",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@supabase/supabase-js": "^2.26.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "supabase-management-js": "^1.0.0",
     "zod": "3.22.3"
   },

integrations/typeform/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @trigger.dev/typeform
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.4
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+  - @trigger.dev/integration-kit@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.2
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+  - @trigger.dev/integration-kit@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

integrations/typeform/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/typeform",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "description": "The official Typeform integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.4",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4",
     "@typeform/api-client": "^1.8.0",
     "zod": "3.22.3"
   },

package.json

@@ -38,7 +38,7 @@
     "env:pull": "turbo run env:pull",
     "changeset:add": "changeset",
     "changeset:version": "changeset version",
-    "changeset:release": "pnpm run build --filter \"@trigger.dev/*\" && changeset publish",
+    "changeset:release": "pnpm run build --filter \"@trigger.dev/*\" --filter \"trigger.dev\" && changeset publish",
     "changeset:beta": "changeset pre enter beta",
     "changeset:normal": "changeset pre exit",
     "clean:sourcemaps": "turbo run clean:sourcemaps",

packages/astro/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @trigger.dev/astro
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.4
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- Updated dependencies [b271742dc]
+  - @trigger.dev/sdk@3.0.0-beta.3
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.2
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/sdk@3.0.0-beta.1
+
 ## 3.0.0-beta.0
 
 ### Patch Changes

packages/astro/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@trigger.dev/astro",
   "description": "An Astro-native integration for Trigger.dev background jobs platform",
-  "version": "3.0.0-beta.0",
+  "version": "3.0.0-beta.4",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -20,7 +20,7 @@
     "build:tsup": "tsup"
   },
   "peerDependencies": {
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.4"
   },
   "devDependencies": {
     "astro": "^3.0.12",

packages/cli-v3/CHANGELOG.md

@@ -1,5 +1,47 @@
 # trigger.dev
 
+## 3.0.0-beta.4
+
+### Patch Changes
+
+- c702d6a9c: better handle task metadata parse errors, and display nicely formatted errors
+- b271742dc: Configurable log levels in the config file and via env var
+- 8c4df326c: Improve error messages during dev/deploy and handle deploy image build issues
+- b271742dc: Added a Node.js runtime check for the CLI
+- 8694e573f: Fix CLI logout and add list-profiles command
+- Updated dependencies [c702d6a9c]
+- Updated dependencies [b271742dc]
+- Updated dependencies [9af2570da]
+  - @trigger.dev/core@3.0.0-beta.3
+
+## 3.0.0-beta.3
+
+### Patch Changes
+
+- e3cf456c6: Handle string and non-stringifiable outputs like functions
+- Updated dependencies [e3cf456c6]
+  - @trigger.dev/core@3.0.0-beta.2
+
+## 3.0.0-beta.2
+
+### Patch Changes
+
+- b66d5525e: add machine config and secure zod connection
+- d9c9e80bc: Changed "Worker" to "Version" in the dev command key
+- 719c0a0b9: Fixed incorrect span timings around checkpoints by implementing a precise wall clock that resets after restores
+- 9bcb8cb42: Added DEBUG to the ignored env vars
+- f93eae300: Dynamically import superjson and fix some bundling issues
+- Updated dependencies [b66d5525e]
+- Updated dependencies [719c0a0b9]
+- Updated dependencies [f93eae300]
+  - @trigger.dev/core@3.0.0-beta.1
+
+## 3.0.0-beta.1
+
+### Patch Changes
+
+- f1571cbfa: Fixed an issue where the trigger.dev package was not being built before publishing to npm
+
 ## 3.0.0-beta.0
 
 ### Major Changes