WIP (fly.io multi-tenant idea)

apps/webapp/app/env.server.ts

@@ -40,6 +40,14 @@ const EnvironmentSchema = z.object({
   EXECUTION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   WORKER_ENABLED: z.string().default("true"),
   EXECUTION_WORKER_ENABLED: z.string().default("true"),
+  // Docker Registry
+  DOCKER_REGISTRY_HOST: z.string().optional(),
+  DOCKER_REGISTRY_USERNAME: z.string().optional(),
+  DOCKER_REGISTRY_PASSWORD: z.string().optional(),
+  // Fly Background Task Provider
+  FLY_IO_API_TOKEN: z.string().optional(),
+  FLY_IO_API_URL: z.string().url().optional(),
+  FLY_IO_ORG_SLUG: z.string().optional(),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/models/backgroundTaskSecret.server.ts

@@ -1,7 +1,16 @@
-import { BackgroundTaskVersion } from "@trigger.dev/database";
+import {
+  BackgroundTaskSecret,
+  BackgroundTaskVersion,
+  SecretReference,
+} from "@trigger.dev/database";
+import { z } from "zod";
 import { $transaction, PrismaClientOrTransaction } from "~/db.server";
 import { SecretStore, getSecretStore } from "~/services/secrets/secretStore.server";
 
+const BackgroundTaskSecretSchema = z.object({
+  secret: z.string(),
+});
+
 export async function createBackgroundTaskSecret(
   prisma: PrismaClientOrTransaction,
   version: BackgroundTaskVersion,
@@ -116,3 +125,12 @@ export async function deleteBackgroundTaskSecret(prisma: PrismaClientOrTransacti
     await secretStore.deleteSecret(secret.secretReference.key);
   });
 }
+
+export async function resolveBackgroundTaskSecret(reference: SecretReference) {
+  const secretStoreProvider = getSecretStore(reference.provider);
+  const secretStore = new SecretStore(secretStoreProvider);
+
+  const secretRecord = await secretStore.getSecret(BackgroundTaskSecretSchema, reference.key);
+
+  return secretRecord?.secret;
+}

apps/webapp/app/models/task.server.ts

@@ -1,5 +1,6 @@
 import type { Task, TaskAttempt } from "@trigger.dev/database";
 import { ServerTask } from "@trigger.dev/core";
+import { PrismaClientOrTransaction } from "~/db.server";
 
 export type TaskWithAttempts = Task & { attempts: TaskAttempt[] };
 
@@ -25,3 +26,20 @@ export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask
     operation: task.operation,
   };
 }
+
+export type KitchenSinkTask = NonNullable<Awaited<ReturnType<typeof findKitchenSinkTask>>>;
+
+export async function findKitchenSinkTask(prisma: PrismaClientOrTransaction, id: string) {
+  return prisma.task.findUnique({
+    where: { id },
+    include: {
+      attempts: true,
+      run: {
+        include: {
+          environment: true,
+          queue: true,
+        },
+      },
+    },
+  });
+}

apps/webapp/app/routes/_docker.v2.$.ts

@@ -0,0 +1,10 @@
+import { ActionArgs, LoaderArgs } from "@remix-run/server-runtime";
+import { proxyToRegistry } from "~/services/docker/registryProxy.server";
+
+export async function action({ request }: ActionArgs) {
+  return await proxyToRegistry(request);
+}
+
+export async function loader({ request, params }: LoaderArgs) {
+  return await proxyToRegistry(request);
+}

apps/webapp/app/routes/_docker.v2.ts

@@ -0,0 +1,6 @@
+import { LoaderArgs } from "@remix-run/server-runtime";
+import { proxyToRegistry } from "~/services/docker/registryProxy.server";
+
+export async function loader({ request }: LoaderArgs) {
+  return await proxyToRegistry(request);
+}

apps/webapp/app/routes/api.v1.background.artifacts.$id.images.ts

@@ -0,0 +1,66 @@
+import type { ActionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { CreateBackgroundTaskImageRequestBodySchema } from "@trigger.dev/core";
+import { z } from "zod";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CreateBackgroundTaskImageService } from "~/services/backgroundTasks/createBackgroundTaskImage.server";
+
+const ParamsSchema = z.object({
+  id: z.string(),
+});
+
+export async function action({ request, params }: ActionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid request params" }, { status: 400 });
+  }
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  const body = CreateBackgroundTaskImageRequestBodySchema.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new CreateBackgroundTaskImageService();
+
+  try {
+    const image = await service.call(
+      authenticationResult.environment,
+      parsedParams.data.id,
+      body.data
+    );
+
+    if (!image) {
+      return json(
+        {
+          error: `Unable to create background task image`,
+        },
+        { status: 500 }
+      );
+    }
+
+    return json(image);
+  } catch (error) {
+    if (error instanceof Error) {
+      return json({ error: error.message }, { status: 400 });
+    }
+
+    return json({ error: "Something went wrong" }, { status: 500 });
+  }
+}

apps/webapp/app/routes/api.v1.background.artifacts.$id[.tgz].ts

@@ -0,0 +1,142 @@
+import { Response } from "@remix-run/node";
+import { LoaderArgs, json } from "@remix-run/server-runtime";
+import { BackgroundTask, BackgroundTaskArtifact, PrismaClient } from "@trigger.dev/database";
+import archiver from "archiver";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+
+const ParamsSchema = z.object({
+  id: z.string(),
+});
+
+export async function loader({ params }: LoaderArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid request params" }, { status: 400 });
+  }
+
+  const service = new GenerateBackgroundTaskArtifactArchiveService();
+
+  const results = await service.call(parsedParams.data.id);
+
+  if (!results) {
+    return json({ error: "Something went wrong" }, { status: 500 });
+  }
+
+  return new Response(results.archive, {
+    headers: {
+      "Content-Disposition": `attachment; filename="${results.name}"`,
+      "Content-Type": "application/gzip",
+    },
+  });
+}
+
+class GenerateBackgroundTaskArtifactArchiveService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const artifact = await this.#prismaClient.backgroundTaskArtifact.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        backgroundTask: true,
+      },
+    });
+
+    if (!artifact) {
+      return;
+    }
+
+    const archive = archiver("tar", {
+      gzip: true,
+      zlib: { level: 9 }, // Sets the compression level
+    });
+
+    function addFileToContext(contents: string, path: string) {
+      // Append files to the archive
+      archive.append(contents, { name: `ctx/${path}` });
+    }
+
+    // Good practice to catch warnings (ie stat failures and other non-blocking errors)
+    archive.on("warning", function (err) {
+      if (err.code === "ENOENT") {
+        // log warning
+        console.warn(err);
+      } else {
+        // throw error
+        throw err;
+      }
+    });
+
+    // Good practice to catch this error explicitly
+    archive.on("error", function (err) {
+      throw err;
+    });
+
+    addFileToContext(artifact.bundle, `src/${artifact.fileName}`);
+    addFileToContext(
+      JSON.stringify(this.#generatePackageJson(artifact, artifact.backgroundTask)),
+      "package.json"
+    );
+    addFileToContext(this.#generateDockerfile(artifact, artifact.backgroundTask), "Dockerfile");
+    addFileToContext(this.#generateIndexJs(artifact, artifact.backgroundTask), "src/index.js");
+
+    // Finalize the archive
+    archive.finalize();
+
+    return {
+      archive,
+      name: `${artifact.id}.tar.gz`,
+    };
+  }
+
+  #generatePackageJson(artifact: BackgroundTaskArtifact, task: BackgroundTask) {
+    return {
+      name: task.slug,
+      version: artifact.version,
+      description: `Trigger background task ${task.slug}`,
+      main: "src/index.js",
+      scripts: {
+        start: "node src/index.js",
+      },
+      dependencies: artifact.dependencies,
+      engines: {
+        node: this.#getNodeVersion(artifact.nodeVersion),
+      },
+    };
+  }
+
+  #generateDockerfile(artifact: BackgroundTaskArtifact, task: BackgroundTask) {
+    return `FROM amd64/node:${this.#getNodeVersion(artifact.nodeVersion)}-bullseye-slim
+    
+    WORKDIR /usr/src/app
+    
+    COPY ctx/package*.json ./
+    
+    RUN npm install
+    
+    COPY ctx/. .
+    
+    CMD [ "npm", "start" ]
+    `;
+  }
+
+  #generateIndexJs(artifact: BackgroundTaskArtifact, task: BackgroundTask) {
+    return `
+      const task = require("./${artifact.fileName}").default; 
+      console.log(task);
+      console.log(process.env);
+    `;
+  }
+
+  // replace the v if it exists
+  #getNodeVersion(version: string) {
+    return version.replace("v", "");
+  }
+}

apps/webapp/app/routes/api.v1.background.tasks.ts

@@ -29,9 +29,9 @@ export async function action({ request, params }: ActionArgs) {
   const service = new DeployBackgroundTaskService();
 
   try {
-    const artifact = await service.call(authenticationResult.environment, body.data);
+    const results = await service.call(authenticationResult.environment, body.data);
 
-    if (!artifact) {
+    if (!results) {
       return json(
         {
           error: `Unable to deploy background task, Task with ID = ${body.data.id} not found`,
@@ -40,9 +40,13 @@ export async function action({ request, params }: ActionArgs) {
       );
     }
 
+    const { artifact, imageConfig } = results;
+
     return json({
       id: artifact.id,
       hash: artifact.hash,
+      image: imageConfig.image,
+      tag: imageConfig.tag,
       createdAt: artifact.createdAt,
       updatedAt: artifact.updatedAt,
     });

apps/webapp/app/services/apiAuth.server.ts

@@ -26,30 +26,44 @@ export async function authenticateApiRequest(
     return;
   }
 
+  return await authenticateApiKey(result, { allowPublicKey });
+}
+
+export async function authenticateApiKey(
+  apiKey: string,
+  { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
+): Promise<ApiAuthenticationResult | undefined> {
+  const type = isPublicApiKey(apiKey) ? ("PUBLIC" as const) : ("PRIVATE" as const);
+
   //if it's a public API key and we don't allow public keys, return
   if (!allowPublicKey) {
-    const environment = await findEnvironmentByApiKey(result.apiKey);
+    const environment = await findEnvironmentByApiKey(apiKey);
     if (!environment) return;
+
     return {
-      ...result,
+      apiKey,
+      type,
       environment,
     };
   }
 
-  switch (result.type) {
+  switch (type) {
     case "PUBLIC": {
-      const environment = await findEnvironmentByPublicApiKey(result.apiKey);
+      const environment = await findEnvironmentByPublicApiKey(apiKey);
       if (!environment) return;
       return {
-        ...result,
+        apiKey,
+        type,
         environment,
       };
     }
     case "PRIVATE": {
-      const environment = await findEnvironmentByApiKey(result.apiKey);
+      const environment = await findEnvironmentByApiKey(apiKey);
       if (!environment) return;
+
       return {
-        ...result,
+        apiKey,
+        type,
         environment,
       };
     }
@@ -69,6 +83,6 @@ export function getApiKeyFromRequest(request: Request) {
   }
 
   const apiKey = authorization.data.replace(/^Bearer /, "");
-  const type = isPublicApiKey(apiKey) ? ("PUBLIC" as const) : ("PRIVATE" as const);
-  return { apiKey, type };
+
+  return apiKey;
 }

apps/webapp/app/services/backgroundTasks/assignOperationToPool.server.ts

@@ -0,0 +1,66 @@
+import {
+  BackgroundTaskImage,
+  BackgroundTaskOperation,
+  BackgroundTaskVersion,
+} from "@trigger.dev/database";
+import type { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
+import { $transaction, prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { AutoScalePoolService } from "./autoScalePool.server";
+
+export class AssignOperationToPoolService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    operation: BackgroundTaskOperation,
+    version: BackgroundTaskVersion,
+    image: BackgroundTaskImage
+  ) {
+    return await $transaction(this.#prismaClient, async (tx) => {
+      const pool = await tx.backgroundTaskMachinePool.upsert({
+        where: {
+          backgroundTaskVersionId_imageId: {
+            backgroundTaskVersionId: version.id,
+            imageId: image.id,
+          },
+        },
+        create: {
+          backgroundTaskVersionId: version.id,
+          imageId: image.id,
+          backgroundTaskId: operation.backgroundTaskId,
+          provider: image.provider,
+          region: version.region,
+          cpu: version.cpu,
+          memory: version.memory,
+          concurrency: version.concurrency,
+          diskSize: version.diskSize,
+        },
+        update: {
+          region: version.region,
+          cpu: version.cpu,
+          memory: version.memory,
+          concurrency: version.concurrency,
+          diskSize: version.diskSize,
+        },
+      });
+
+      const updatedOperation = await tx.backgroundTaskOperation.update({
+        where: {
+          id: operation.id,
+        },
+        data: {
+          status: "ASSIGNED_TO_POOL",
+          poolId: pool.id,
+        },
+      });
+
+      await AutoScalePoolService.enqueue(pool, tx, true);
+
+      return updatedOperation;
+    });
+  }
+}

apps/webapp/app/services/backgroundTasks/autoScalePool.server.ts

@@ -0,0 +1,191 @@
+import {
+  BackgroundTask,
+  BackgroundTaskMachine,
+  BackgroundTaskMachinePool,
+} from "@trigger.dev/database";
+import type { PrismaClientOrTransaction } from "~/db.server";
+import { prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { backgroundTaskProvider } from "./provider.server";
+import { CreateExternalMachineService } from "./createExternalMachine.server";
+
+const frequency = 1000 * 30; // 30 seconds
+
+export class AutoScalePoolService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const pool = await this.#prismaClient.backgroundTaskMachinePool.findUniqueOrThrow({
+      where: {
+        id,
+      },
+      include: {
+        _count: {
+          select: {
+            operations: {
+              where: {
+                status: "ASSIGNED_TO_POOL",
+              },
+            },
+          },
+        },
+        machines: true,
+        backgroundTask: true,
+        backgroundTaskVersion: true,
+      },
+    });
+
+    // Index the machines to gather the current state of the pool
+
+    // If there are no operations, we can re-enqueue this job
+    if (pool._count.operations === 0) {
+      return await AutoScalePoolService.enqueue(pool, this.#prismaClient);
+    }
+
+    // TODO: we probably should just list all the machines for the app?
+
+    const machines = await this.#autoScaleMachines(
+      pool.backgroundTaskVersion.concurrency,
+      pool,
+      pool.machines,
+      pool.backgroundTask
+    );
+
+    // We need to create any pending machines
+    const pendingMachines = machines.filter((machine) => machine.status === "PENDING");
+
+    for (const pendingMachine of pendingMachines) {
+      await CreateExternalMachineService.enqueue(pendingMachine, this.#prismaClient);
+    }
+
+    await backgroundTaskProvider.cleanupForTask(pool.backgroundTask);
+
+    await AutoScalePoolService.enqueue(pool, this.#prismaClient);
+  }
+
+  // If there are operations, we need to scale up the pool
+  // Machines will automatically be restarted when they are returned to the pool
+  // So we just need to make sure at least one machine is running
+  // And if there is not, we need to start one
+  // Status
+  // machine statutes:
+  // PENDING - The record has been created, but the machine has not on the provider
+  // CREATED - The machine has been created on the provider
+  // STARTING
+  // STARTED - The machine is running
+  // STOPPING
+  // STOPPED
+  // DESTROYING
+  // DESTROYED - The machine has been destroyed on the provider
+  // REPLACING - The machine config is being updated on the provider
+  async #autoScaleMachines(
+    target: number,
+    pool: BackgroundTaskMachinePool,
+    existingMachines: BackgroundTaskMachine[],
+    task: BackgroundTask
+  ): Promise<BackgroundTaskMachine[]> {
+    const pendingMachines: BackgroundTaskMachine[] = [];
+
+    // We need to update the pool to have the correct number of machines
+    if (existingMachines.length < target) {
+      const machinesToCreate = target - existingMachines.length;
+
+      for (let i = 0; i < machinesToCreate; i++) {
+        const pendingMachine = await this.#prismaClient.backgroundTaskMachine.create({
+          data: {
+            provider: backgroundTaskProvider.name,
+            poolId: pool.id,
+            backgroundTaskId: pool.backgroundTaskId,
+            backgroundTaskVersionId: pool.backgroundTaskVersionId,
+            backgroundTaskImageId: pool.imageId,
+          },
+        });
+
+        pendingMachines.push(pendingMachine);
+      }
+    }
+
+    const updatedExistingMachines = (
+      await Promise.all(existingMachines.map(async (machine) => this.#indexMachine(machine, task)))
+    ).filter(Boolean);
+
+    const replacedMachines: BackgroundTaskMachine[] = [];
+
+    if (updatedExistingMachines.length < existingMachines.length) {
+      const machinesToReplace = existingMachines.length - updatedExistingMachines.length;
+
+      for (let i = 0; i < machinesToReplace; i++) {
+        const pendingMachine = await this.#prismaClient.backgroundTaskMachine.create({
+          data: {
+            provider: backgroundTaskProvider.name,
+            poolId: pool.id,
+            backgroundTaskId: pool.backgroundTaskId,
+            backgroundTaskVersionId: pool.backgroundTaskVersionId,
+            backgroundTaskImageId: pool.imageId,
+          },
+        });
+
+        replacedMachines.push(pendingMachine);
+      }
+    }
+
+    return [...pendingMachines, ...updatedExistingMachines, ...replacedMachines];
+  }
+
+  async #indexMachine(
+    machine: BackgroundTaskMachine,
+    task: BackgroundTask
+  ): Promise<BackgroundTaskMachine | undefined> {
+    // Using the provider get updated information about the machine (if it has an externalId)
+    if (!machine.externalId) {
+      return machine;
+    }
+
+    const externalMachine = await backgroundTaskProvider.getMachineForTask(
+      machine.externalId,
+      task
+    );
+
+    if (!externalMachine) {
+      await this.#prismaClient.backgroundTaskMachine.delete({
+        where: {
+          id: machine.id,
+        },
+      });
+
+      return;
+    }
+
+    return await this.#prismaClient.backgroundTaskMachine.update({
+      where: {
+        id: machine.id,
+      },
+      data: {
+        status: externalMachine.status,
+        data: externalMachine.data,
+      },
+    });
+  }
+
+  static async enqueue(
+    pool: BackgroundTaskMachinePool,
+    tx: PrismaClientOrTransaction = prisma,
+    force = false
+  ) {
+    return await workerQueue.enqueue(
+      "autoScalePool",
+      {
+        id: pool.id,
+      },
+      {
+        tx,
+        jobKey: `scale:${pool.id}`,
+        runAt: force ? new Date() : new Date(Date.now() + frequency),
+      }
+    );
+  }
+}

apps/webapp/app/services/backgroundTasks/createBackgroundTaskImage.server.ts

@@ -0,0 +1,62 @@
+import { CreateBackgroundTaskImageRequestBody } from "@trigger.dev/core";
+import { PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+import { backgroundTaskProvider } from "./provider.server";
+
+export class CreateBackgroundTaskImageService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    environment: AuthenticatedEnvironment,
+    id: string,
+    payload: CreateBackgroundTaskImageRequestBody
+  ) {
+    // Find the artifact
+    const artifact = await this.#prismaClient.backgroundTaskArtifact.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        backgroundTask: true,
+      },
+    });
+
+    if (!artifact) {
+      return;
+    }
+
+    if (artifact.backgroundTask.projectId !== environment.projectId) {
+      return;
+    }
+
+    const image = await this.#prismaClient.backgroundTaskImage.upsert({
+      where: {
+        backgroundTaskArtifactId_digest: {
+          backgroundTaskArtifactId: artifact.id,
+          digest: payload.digest,
+        },
+      },
+      create: {
+        backgroundTaskArtifactId: artifact.id,
+        backgroundTaskId: artifact.backgroundTaskId,
+        digest: payload.digest,
+        name: payload.name,
+        tag: payload.tag,
+        size: payload.size,
+        provider: backgroundTaskProvider.name,
+      },
+      update: {
+        name: payload.name,
+        tag: payload.tag,
+        size: payload.size,
+      },
+    });
+
+    return image;
+  }
+}

apps/webapp/app/services/backgroundTasks/createExternalMachine.server.ts

@@ -0,0 +1,98 @@
+import { BackgroundTaskMachine } from "@trigger.dev/database";
+import type { PrismaClientOrTransaction } from "~/db.server";
+import { prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { backgroundTaskProvider } from "./provider.server";
+import { resolveBackgroundTaskSecret } from "~/models/backgroundTaskSecret.server";
+import { env } from "~/env.server";
+import { ExternalMachineConfig } from "./providers/types";
+
+export class CreateExternalMachineService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const machine = await this.#prismaClient.backgroundTaskMachine.findUniqueOrThrow({
+      where: {
+        id,
+      },
+      include: {
+        pool: {
+          include: {
+            image: true,
+            backgroundTask: true,
+            backgroundTaskVersion: {
+              include: {
+                environment: true,
+                secrets: {
+                  include: {
+                    secretReference: true,
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    const envVars: Record<string, string> = {};
+
+    for (const secret of machine.pool.backgroundTaskVersion.secrets) {
+      const secretValue = await resolveBackgroundTaskSecret(secret.secretReference);
+
+      if (!secretValue) {
+        continue;
+      }
+
+      envVars[secret.key] = secretValue;
+    }
+
+    envVars["TRIGGER_API_KEY"] = machine.pool.backgroundTaskVersion.environment.apiKey;
+    envVars["TRIGGER_API_URL"] = env.APP_ORIGIN;
+    envVars["TRIGGER_POOL_ID"] = machine.pool.id;
+    envVars["TRIGGER_MACHINE_ID"] = machine.id;
+
+    const config: ExternalMachineConfig = {
+      cpus: machine.pool.cpu,
+      memory: machine.pool.memory,
+      diskSize: machine.pool.diskSize,
+      region: machine.pool.region,
+      env: envVars,
+      image: `${backgroundTaskProvider.registry}/${machine.pool.image.name}:${machine.pool.image.tag}@${machine.pool.image.digest}`,
+    };
+
+    const externalMachine = await backgroundTaskProvider.createMachineForTask(
+      machine.id,
+      machine.pool.backgroundTask,
+      config
+    );
+
+    await this.#prismaClient.backgroundTaskMachine.update({
+      where: {
+        id: machine.id,
+      },
+      data: {
+        externalId: externalMachine.id,
+        status: externalMachine.status,
+        data: externalMachine.data,
+      },
+    });
+  }
+
+  static async enqueue(machine: BackgroundTaskMachine, tx: PrismaClientOrTransaction = prisma) {
+    return await workerQueue.enqueue(
+      "createExternalMachine",
+      {
+        id: machine.id,
+      },
+      {
+        tx,
+        jobKey: `createMachine:${machine.id}`,
+      }
+    );
+  }
+}

apps/webapp/app/services/backgroundTasks/deployBackgroundTask.server.ts

@@ -1,8 +1,9 @@
-import { BackgroundTaskArtifact, PrismaClient } from "@trigger.dev/database";
-import { AuthenticatedEnvironment } from "../apiAuth.server";
 import { DeployBackgroundTaskRequestBody } from "@trigger.dev/core";
-import { prisma } from "~/db.server";
+import { PrismaClient } from "@trigger.dev/database";
 import nodeCrypto from "node:crypto";
+import { prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+import { backgroundTaskProvider } from "./provider.server";
 
 export class DeployBackgroundTaskService {
   #prismaClient: PrismaClient;
@@ -14,7 +15,7 @@ export class DeployBackgroundTaskService {
   public async call(
     environment: AuthenticatedEnvironment,
     payload: DeployBackgroundTaskRequestBody
-  ): Promise<BackgroundTaskArtifact | undefined> {
+  ) {
     const hash = this.#hashPayload(payload);
 
     const backgroundTask = await this.#prismaClient.backgroundTask.findUnique({
@@ -30,7 +31,7 @@ export class DeployBackgroundTaskService {
       return;
     }
 
-    return this.#prismaClient.backgroundTaskArtifact.upsert({
+    const artifact = await this.#prismaClient.backgroundTaskArtifact.upsert({
       where: {
         backgroundTaskId_version_hash: {
           backgroundTaskId: backgroundTask.id,
@@ -56,6 +57,13 @@ export class DeployBackgroundTaskService {
         sourcemap: payload.sourcemap,
       },
     });
+
+    const imageConfig = await backgroundTaskProvider.prepareArtifact(backgroundTask, artifact);
+
+    return {
+      artifact,
+      imageConfig,
+    };
   }
 
   #hashPayload(payload: DeployBackgroundTaskRequestBody) {

apps/webapp/app/services/backgroundTasks/executeBackgroundTaskOperation.server.ts

@@ -0,0 +1,55 @@
+import { PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { AssignOperationToPoolService } from "./assignOperationToPool.server";
+
+export class ExecuteBackgroundTaskOperationService {
+  #prismaClient: PrismaClient;
+  #assignOperationToPoolService = new AssignOperationToPoolService();
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const operation = await this.#prismaClient.backgroundTaskOperation.findUniqueOrThrow({
+      where: {
+        id,
+      },
+      include: {
+        backgroundTask: true,
+        backgroundTaskVersion: true,
+      },
+    });
+
+    // Find the BackgroundTaskImage
+    const image = await this.#prismaClient.backgroundTaskImage.findFirst({
+      where: {
+        backgroundTaskId: operation.backgroundTaskId,
+        tag: operation.backgroundTaskVersion.version,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+
+    // If the image is not found, we need to wait for it to be deployed
+    if (!image) {
+      await this.#prismaClient.backgroundTaskOperation.update({
+        where: {
+          id,
+        },
+        data: {
+          status: "WAITING_ON_IMAGE",
+        },
+      });
+
+      return;
+    }
+
+    return await this.#assignOperationToPoolService.call(
+      operation,
+      operation.backgroundTaskVersion,
+      image
+    );
+  }
+}

apps/webapp/app/services/backgroundTasks/initializeBackgroundTaskOperation.server.ts

@@ -0,0 +1,104 @@
+import { BackgroundTaskOperationParamsSchema } from "@trigger.dev/core";
+import { PrismaClient, RuntimeEnvironmentType, Task } from "@trigger.dev/database";
+import { $transaction, prisma } from "~/db.server";
+import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
+import { KitchenSinkTask } from "~/models/task.server";
+import { workerQueue } from "../worker.server";
+
+export class InitializeBackgroundTaskOperationService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(task: KitchenSinkTask) {
+    const params = BackgroundTaskOperationParamsSchema.safeParse(task.params);
+    // We need to create a new background task operation
+
+    if (!params.success) {
+      await this.#resumeTaskWithError(task, params.error.message);
+      return;
+    }
+
+    const backgroundTask = await this.#prismaClient.backgroundTask.findUnique({
+      where: {
+        projectId_slug: {
+          projectId: task.run.projectId,
+          slug: params.data.id,
+        },
+      },
+      include: {
+        versions: {
+          where: {
+            version: params.data.version,
+          },
+        },
+      },
+    });
+
+    if (!backgroundTask) {
+      await this.#resumeTaskWithError(task, `Background task ${params.data.id} not found`);
+      return;
+    }
+
+    const version = backgroundTask.versions[0];
+
+    if (!version) {
+      await this.#resumeTaskWithError(
+        task,
+        `Background task ${params.data.id} version ${params.data.version} not found`
+      );
+      return;
+    }
+
+    await $transaction(this.#prismaClient, async (tx) => {
+      const operation = await tx.backgroundTaskOperation.create({
+        data: {
+          backgroundTaskId: backgroundTask.id,
+          backgroundTaskVersionId: version.id,
+          taskId: task.id,
+          payload: params.data.payload,
+        },
+      });
+
+      await workerQueue.enqueue(
+        "executeBackgroundTaskOperation",
+        {
+          id: operation.id,
+        },
+        { tx }
+      );
+
+      return operation;
+    });
+  }
+
+  async #resumeTaskWithError(task: KitchenSinkTask, message: string) {
+    await $transaction(this.#prismaClient, async (tx) => {
+      await tx.task.update({
+        where: { id: task.id },
+        data: {
+          status: "ERRORED",
+          completedAt: new Date(),
+          output: { message },
+        },
+      });
+
+      await tx.taskAttempt.updateMany({
+        where: {
+          taskId: task.id,
+          status: "PENDING",
+        },
+        data: {
+          status: "ERRORED",
+          error: message,
+        },
+      });
+
+      await enqueueRunExecutionV2(task.run, prisma, {
+        skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
+      });
+    });
+  }
+}

apps/webapp/app/services/backgroundTasks/provider.server.ts

@@ -0,0 +1,56 @@
+import { BackgroundTask, BackgroundTaskProviderStrategy } from "@trigger.dev/database";
+import { env } from "~/env.server";
+import { FlyBackgroundTaskProvider } from "./providers/fly.server";
+import { BackgroundTaskProvider, ExternalMachine, ExternalMachineConfig } from "./providers/types";
+
+export class UnsupportedBackgroundTaskProvider implements BackgroundTaskProvider {
+  async prepareArtifact(task: BackgroundTask): Promise<any> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  get name(): BackgroundTaskProviderStrategy {
+    return "UNSUPPORTED";
+  }
+
+  get defaultRegion(): string {
+    return "UNSUPPORTED";
+  }
+
+  get registry(): string {
+    return "UNSUPPORTED";
+  }
+
+  async getMachineForTask(id: string, task: BackgroundTask): Promise<ExternalMachine | undefined> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  getMachinesForTask(task: BackgroundTask): Promise<Array<ExternalMachine>> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  createMachineForTask(
+    id: string,
+    task: BackgroundTask,
+    config: ExternalMachineConfig
+  ): Promise<ExternalMachine> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  cleanupForTask(task: BackgroundTask): Promise<void> {
+    throw new Error("Unsupported background task provider");
+  }
+}
+
+let backgroundTaskProvider: BackgroundTaskProvider;
+
+if (env.FLY_IO_API_TOKEN && env.FLY_IO_API_URL && env.FLY_IO_ORG_SLUG) {
+  backgroundTaskProvider = new FlyBackgroundTaskProvider(
+    env.FLY_IO_API_URL,
+    env.FLY_IO_ORG_SLUG,
+    env.FLY_IO_API_TOKEN
+  );
+} else {
+  backgroundTaskProvider = new UnsupportedBackgroundTaskProvider();
+}
+
+export { backgroundTaskProvider };

apps/webapp/app/services/backgroundTasks/providers/fly.server.ts

@@ -0,0 +1,569 @@
+import {
+  BackgroundTask,
+  BackgroundTaskArtifact,
+  BackgroundTaskMachineStatus,
+  BackgroundTaskProviderStrategy,
+} from "@trigger.dev/database";
+import { z } from "zod";
+import { logger } from "~/services/logger.server";
+import { ZodResponse, zodfetch } from "~/zodfetch.server";
+import { BackgroundTaskProvider, ExternalMachine, ExternalMachineConfig } from "./types";
+import retry from "async-retry";
+import AsyncRetry from "async-retry";
+
+const FlyAppSchema = z.object({
+  name: z.string(),
+  organization: z.object({
+    name: z.string(),
+    slug: z.string(),
+  }),
+  status: z.string(),
+});
+
+const FlyCreateAppSchema = z.object({
+  app_name: z.string(),
+  org_slug: z.string(),
+  network: z.string().optional(),
+});
+
+const FlyCheckStatusSchema = z.object({
+  name: z.string(),
+  output: z.string(),
+  status: z.string(),
+  updated_at: z.coerce.date().optional(),
+});
+
+const FlyMachineGuestSchema = z.object({
+  cpu_kind: z.enum(["shared", "dedicated"]),
+  cpus: z.number(),
+  memory_mb: z.number(),
+});
+
+const FlyMachineMetricsSchema = z.object({
+  path: z.string(),
+  port: z.number(),
+});
+
+const FlyMachineMountSchema = z.object({
+  encrypted: z.boolean().optional(),
+  name: z.string().optional(),
+  path: z.string(),
+  volume: z.string(),
+  size_gb: z.number().optional(),
+});
+
+const FlyMachineProcessSchema = z.object({
+  cmd: z.array(z.string()).optional(),
+  entrypoint: z.string().optional(),
+  env: z.record(z.string()).default({}),
+  exec: z.array(z.string()).optional(),
+  user: z.string().optional(),
+});
+
+const FlyMachineRestartSchema = z.object({
+  max_retries: z.number().optional(),
+  policy: z.string().optional(),
+});
+
+const FlyMachineHTTPHeaderSchema = z.array(
+  z.object({ name: z.string(), values: z.array(z.string()) })
+);
+
+const FlyMachineCheckSchema = z.object({
+  grace_period: z.string().optional(),
+  headers: z.array(FlyMachineHTTPHeaderSchema).default([]),
+  interval: z.string().optional(),
+  method: z.string().optional(),
+  path: z.string().optional(),
+  port: z.number().optional(),
+  protocol: z.string().optional(),
+  timeouyt: z.string().optional(),
+  tls_server_name: z.string().optional(),
+  tls_skip_verify: z.boolean().optional(),
+  type: z.string().optional(),
+});
+
+const FlyMachineServiceConcurrencySchema = z.object({
+  hard_limit: z.number().optional(),
+  soft_limit: z.number().optional(),
+  type: z.string(),
+});
+
+const FlyHTTPOptionsSchema = z.object({
+  compress: z.boolean().optional(),
+  response: z
+    .object({
+      headers: z.record(z.string()).default({}),
+    })
+    .optional(),
+});
+
+const FlyMachinePortSchema = z.object({
+  end_port: z.number(),
+  force_https: z.boolean().optional(),
+  handlers: z.array(z.string()).default([]),
+  http_options: FlyHTTPOptionsSchema.optional(),
+});
+
+const FlyMachineServiceSchema = z.object({
+  autostart: z.boolean().optional(),
+  autostop: z.boolean().optional(),
+  checks: z.array(FlyMachineCheckSchema).default([]),
+  concurrency: FlyMachineServiceConcurrencySchema,
+  force_instance_description: z.string().optional(),
+  force_instance_key: z.string().optional(),
+  internal_port: z.number().optional(),
+  min_machines_running: z.number().optional(),
+  ports: z.array(FlyMachinePortSchema).default([]),
+  protocol: z.string().optional(),
+});
+
+const FlyMachineStaticSchema = z.object({
+  guest_path: z.string(),
+  url_prefix: z.string(),
+});
+
+const FlyMachineStopConfigSchema = z.object({
+  signal: z.string().optional(),
+  timeout: z.string().optional(),
+});
+
+const FlyMachineConfigSchema = z.object({
+  auto_destroy: z.boolean().optional(),
+  env: z.record(z.string()).default({}),
+  checks: z.record(FlyMachineCheckSchema).default({}),
+  metadata: z.record(z.string()).default({}),
+  guest: FlyMachineGuestSchema,
+  image: z.string(),
+  metrics: FlyMachineMetricsSchema.optional(),
+  mounts: z.array(FlyMachineMountSchema).default([]),
+  processes: z.array(FlyMachineProcessSchema).default([]),
+  restart: FlyMachineRestartSchema.default({}),
+  services: z.array(FlyMachineServiceSchema).default([]),
+  standbys: z.array(z.string()).default([]),
+  statics: z.array(FlyMachineStaticSchema).default([]),
+  stop_config: FlyMachineStopConfigSchema.optional(),
+});
+
+const FlyMachineImageRefSchema = z.object({
+  digest: z.string(),
+  registry: z.string(),
+  repository: z.string(),
+  tag: z.string(),
+  labels: z.record(z.string()).nullable().default({}),
+});
+
+const FlyMachineStateSchema = z.enum([
+  "created",
+  "starting",
+  "started",
+  "stopping",
+  "stopped",
+  "destroying",
+  "destroyed",
+  "replacing",
+]);
+
+const FlyMachineEventSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  status: z.string(),
+  source: z.string(),
+  timestamp: z.coerce.date(),
+  request: z.any(),
+});
+
+const FlyMachineSchema = z.object({
+  id: z.string(),
+  instance_id: z.string(),
+  name: z.string(),
+  nonce: z.string().optional(),
+  private_ip: z.string(),
+  region: z.string(),
+  state: FlyMachineStateSchema,
+  config: FlyMachineConfigSchema,
+  checks: z.array(FlyCheckStatusSchema).optional(),
+  events: z.array(FlyMachineEventSchema).default([]),
+  image_ref: FlyMachineImageRefSchema,
+  created_at: z.coerce.date(),
+  updated_at: z.coerce.date().optional(),
+});
+
+const FlyVolumeSchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  state: z.string(),
+  region: z.string(),
+  size_gb: z.number(),
+  encrypted: z.boolean(),
+  created_at: z.coerce.date(),
+  attached_machine_id: z.string().nullable().optional(),
+  attached_alloc_id: z.string().nullable().optional(),
+  blocks: z.number(),
+  block_size: z.number(),
+  blocks_free: z.number(),
+  blocks_avail: z.number(),
+  fstype: z.string(),
+  host_dedication_key: z.string().nullable().optional(),
+});
+
+const FlyCreateVolumeSchema = z.object({
+  name: z.string(),
+  region: z.string(),
+  size_gb: z.number(),
+  machines_only: z.boolean().optional(),
+  encrypted: z.boolean().optional(),
+});
+
+const FlyCreateMachineSchema = z.object({
+  name: z.string(),
+  lease_ttl: z.number().optional(),
+  region: z.string(),
+  config: FlyMachineConfigSchema,
+  skip_launch: z.boolean().optional(),
+  skip_service_registration: z.boolean().optional(),
+});
+
+export class FlyBackgroundTaskProvider implements BackgroundTaskProvider {
+  private readonly _logger = logger.child("FlyBackgroundTaskProvider");
+
+  get name(): BackgroundTaskProviderStrategy {
+    return "FLY_IO";
+  }
+
+  get registry(): string {
+    return "registry.fly.io";
+  }
+
+  constructor(
+    private readonly url: string,
+    private readonly org: string,
+    private readonly token: string
+  ) {}
+
+  get defaultRegion(): string {
+    return "iad";
+  }
+
+  async prepareArtifact(
+    task: BackgroundTask,
+    artifact: BackgroundTaskArtifact
+  ): Promise<{ image: string; tag: string }> {
+    // Check that the app has been created
+    const app = await this.#getApp(this.#appNameForTask(task));
+
+    if (app) {
+      return {
+        image: app.name,
+        tag: artifact.version,
+      };
+    }
+
+    // Create the app
+    const created = await this.#createApp({
+      app_name: this.#appNameForTask(task),
+      network: this.#networkNameForTask(task),
+      org_slug: this.org,
+    });
+
+    if (!created) {
+      throw new Error("Failed to create app");
+    }
+
+    return {
+      image: this.#appNameForTask(task),
+      tag: artifact.version,
+    };
+  }
+
+  async getMachineForTask(id: string, task: BackgroundTask): Promise<ExternalMachine | undefined> {
+    const response = await this.#fetch(
+      FlyMachineSchema,
+      `/v1/apps/${this.#appNameForTask(task)}/machines/${id}`
+    );
+
+    if (!response.ok) {
+      return;
+    }
+
+    return this.#flyMachineToExternalMachine(response.data);
+  }
+
+  async getMachinesForTask(task: BackgroundTask): Promise<Array<ExternalMachine>> {
+    const response = await this.#fetch(
+      z.array(FlyMachineSchema),
+      `/v1/apps/${this.#appNameForTask(task)}/machines`
+    );
+
+    if (!response.ok) {
+      return [];
+    }
+
+    return response.data.map((machine) => this.#flyMachineToExternalMachine(machine));
+  }
+
+  async createMachineForTask(
+    id: string,
+    task: BackgroundTask,
+    config: ExternalMachineConfig
+  ): Promise<ExternalMachine> {
+    // We have to create a volume first
+    const volume = await this.#createVolume(this.#appNameForTask(task), {
+      name: id,
+      region: config.region,
+      size_gb: config.diskSize,
+      encrypted: true,
+      machines_only: true,
+    });
+
+    const machine = await this.#createMachine(this.#appNameForTask(task), {
+      name: id,
+      region: config.region,
+      config: {
+        image: config.image,
+        env: config.env,
+        guest: {
+          cpu_kind: "shared",
+          cpus: config.cpus,
+          memory_mb: config.memory,
+        },
+        auto_destroy: false,
+        mounts: [
+          {
+            volume: volume.id,
+            path: "/data",
+          },
+        ],
+      },
+    });
+
+    return this.#flyMachineToExternalMachine(machine);
+  }
+
+  async cleanupForTask(task: BackgroundTask): Promise<void> {
+    const volumes = await this.#listVolumes(this.#appNameForTask(task));
+
+    if (!volumes) {
+      return;
+    }
+
+    // Destroy any volumn created more than 30 seconds ago that doesn't have a machine attached
+    const destroyableVolumes = volumes.filter(
+      (volume) =>
+        volume.created_at.getTime() < Date.now() - 30 * 1000 &&
+        !volume.attached_machine_id &&
+        !volume.attached_alloc_id &&
+        volume.state !== "pending_destroy"
+    );
+
+    this._logger.debug("cleanupForTask", {
+      volumesToDestroy: destroyableVolumes.length,
+    });
+
+    for (const volume of destroyableVolumes) {
+      await this.#destroyVolume(this.#appNameForTask(task), volume.id);
+    }
+  }
+
+  #flyMachineToExternalMachine(flyMachine: z.output<typeof FlyMachineSchema>): ExternalMachine {
+    return {
+      id: flyMachine.id,
+      status: this.#flyStateToStatus(flyMachine.state),
+      data: flyMachine,
+    };
+  }
+
+  #flyStateToStatus(state: z.infer<typeof FlyMachineStateSchema>): BackgroundTaskMachineStatus {
+    const mappings: Record<z.infer<typeof FlyMachineStateSchema>, BackgroundTaskMachineStatus> = {
+      created: "CREATED",
+      starting: "STARTING",
+      started: "STARTED",
+      stopping: "STOPPING",
+      stopped: "STOPPED",
+      destroying: "DESTROYING",
+      destroyed: "DESTROYED",
+      replacing: "REPLACING",
+    };
+
+    return mappings[state];
+  }
+
+  async #getApp(appName: string) {
+    const response = await this.#fetch(FlyAppSchema, `/v1/apps/${appName}`);
+
+    if (!response.ok) {
+      return;
+    }
+
+    return response.data;
+  }
+
+  async #createApp(body: z.input<typeof FlyCreateAppSchema>) {
+    const response = await this.#fetch(z.any(), "/v1/apps", {
+      method: "POST",
+      body: JSON.stringify(body),
+    });
+
+    return response.ok;
+  }
+
+  async #createMachine(
+    appName: string,
+    body: z.input<typeof FlyCreateMachineSchema>
+  ): Promise<z.output<typeof FlyMachineSchema>> {
+    const response = await this.#fetch(
+      FlyMachineSchema,
+      `/v1/apps/${appName}/machines`,
+      {
+        method: "POST",
+        body: JSON.stringify(body),
+      },
+      {
+        retries: 5,
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error("Failed to create machine");
+    }
+
+    return response.data;
+  }
+
+  async #createVolume(
+    appName: string,
+    body: z.input<typeof FlyCreateVolumeSchema>
+  ): Promise<z.output<typeof FlyVolumeSchema>> {
+    const response = await this.#fetch(
+      FlyVolumeSchema,
+      `/v1/apps/${appName}/volumes`,
+      {
+        method: "POST",
+        body: JSON.stringify(body),
+      },
+      {
+        retries: 5,
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error("Failed to create volume");
+    }
+
+    return response.data;
+  }
+
+  async #listVolumes(
+    appName: string
+  ): Promise<Array<z.output<typeof FlyVolumeSchema>> | undefined> {
+    const response = await this.#fetch(z.array(FlyVolumeSchema), `/v1/apps/${appName}/volumes`, {
+      method: "GET",
+    });
+
+    if (!response.ok) {
+      return;
+    }
+
+    return response.data;
+  }
+
+  async #destroyVolume(appName: string, id: string): Promise<boolean> {
+    const response = await this.#fetch(z.any(), `/v1/apps/${appName}/volumes/${id}`, {
+      method: "DELETE",
+    });
+
+    return response.ok;
+  }
+
+  async #fetch<TResponseSchema extends z.ZodTypeAny>(
+    schema: TResponseSchema,
+    path: string,
+    requestInit?: RequestInit,
+    retryOptions?: AsyncRetry.Options
+  ): Promise<ZodResponse<TResponseSchema>> {
+    const headers = new Headers(requestInit?.headers ?? {});
+
+    // Add the common headers
+    headers.set("Authorization", `Bearer ${this.token}`);
+    headers.set("Accept", "application/json");
+    headers.set("User-Agent", "Trigger.dev/2.1.0");
+
+    if (requestInit?.body) {
+      headers.set("Content-Type", "application/json; charset=utf-8");
+    }
+
+    if (retryOptions) {
+      return await retry(
+        async (bail) => {
+          const response = await zodfetch(schema, `${this.url}${path}`, {
+            ...requestInit,
+            headers,
+          });
+
+          if (!response.ok) {
+            if (response.status === 429 || response.status >= 500) {
+              throw new Error(
+                `[${response.status}] Request ${
+                  requestInit?.method ?? "GET"
+                } ${path} failed: ${JSON.stringify(response.error)}`
+              );
+            }
+
+            bail(
+              new Error(
+                `[${response.status}] Request ${
+                  requestInit?.method ?? "GET"
+                } ${path} failed: ${JSON.stringify(response.error)}`
+              )
+            );
+            return response;
+          }
+
+          return response;
+        },
+        {
+          ...retryOptions,
+          onRetry: (e, attempt) => {
+            this._logger.debug("fetch.retry", {
+              url: `${this.url}${path}`,
+              attempt,
+              response: {
+                ok: response.ok,
+                status: response.status,
+                error: response.ok ? undefined : response.error,
+                err: {
+                  message: e.message,
+                  stack: e.stack,
+                },
+              },
+            });
+          },
+        }
+      );
+    }
+
+    const response = await zodfetch(schema, `${this.url}${path}`, {
+      ...requestInit,
+      headers,
+    });
+
+    this._logger.debug("fetch", {
+      url: `${this.url}${path}`,
+      response: {
+        ok: response.ok,
+        status: response.status,
+        error: response.ok ? undefined : response.error,
+      },
+    });
+
+    return response;
+  }
+
+  #appNameForTask(task: BackgroundTask): string {
+    return `${task.id}-${task.slug}`;
+  }
+
+  #networkNameForTask(task: BackgroundTask): string {
+    return task.projectId;
+  }
+}

apps/webapp/app/services/backgroundTasks/providers/types.ts

@@ -0,0 +1,43 @@
+import type {
+  BackgroundTask,
+  BackgroundTaskArtifact,
+  BackgroundTaskMachineStatus,
+  BackgroundTaskProviderStrategy,
+} from "@trigger.dev/database";
+
+export type ExternalMachine = {
+  id: string;
+  status: BackgroundTaskMachineStatus;
+  data: any;
+};
+
+export type ExternalMachineConfig = {
+  cpus: number;
+  memory: number;
+  diskSize: number;
+  region: string;
+  image: string;
+  env: Record<string, string>;
+};
+
+export interface BackgroundTaskProvider {
+  prepareArtifact(
+    task: BackgroundTask,
+    artifact: BackgroundTaskArtifact
+  ): Promise<{ image: string; tag: string }>;
+
+  get name(): BackgroundTaskProviderStrategy;
+  get defaultRegion(): string;
+  get registry(): string;
+
+  getMachineForTask(id: string, task: BackgroundTask): Promise<ExternalMachine | undefined>;
+  getMachinesForTask(task: BackgroundTask): Promise<Array<ExternalMachine>>;
+
+  createMachineForTask(
+    id: string,
+    task: BackgroundTask,
+    config: ExternalMachineConfig
+  ): Promise<ExternalMachine>;
+
+  cleanupForTask(task: BackgroundTask): Promise<void>;
+}

apps/webapp/app/services/backgroundTasks/registerBackgroundTask.server.ts

@@ -10,6 +10,7 @@ import {
 } from "~/models/backgroundTaskSecret.server";
 import { ExtendedEndpoint, findEndpoint } from "~/models/endpoint.server";
 import type { AuthenticatedEnvironment } from "../apiAuth.server";
+import { backgroundTaskProvider } from "./provider.server";
 
 export class RegisterBackgroundTaskService {
   #prismaClient: PrismaClient;
@@ -114,11 +115,15 @@ export class RegisterBackgroundTaskService {
         cpu: metadata.cpu,
         memory: metadata.memory,
         concurrency: metadata.concurrency ?? DEFAULT_MAX_CONCURRENT_RUNS,
+        region: metadata.region ?? backgroundTaskProvider.defaultRegion,
+        diskSize: metadata.diskSizeInGB,
       },
       update: {
         cpu: metadata.cpu,
         memory: metadata.memory,
         concurrency: metadata.concurrency ?? DEFAULT_MAX_CONCURRENT_RUNS,
+        region: metadata.region,
+        diskSize: metadata.diskSizeInGB,
         endpoint: {
           connect: {
             id: endpoint.id,

apps/webapp/app/services/docker/registryProxy.server.ts

@@ -0,0 +1,146 @@
+import { env } from "~/env.server";
+import { authenticateApiKey } from "../apiAuth.server";
+import { logger } from "../logger.server";
+
+export class RegistryProxy {
+  constructor(public readonly host: string, private auth: { username: string; password: string }) {}
+
+  public async call(request: Request) {
+    return await this.#proxyRequest(request);
+  }
+
+  // Proxies the request to the registry
+  async #proxyRequest(request: Request) {
+    const credentials = this.#getBasicAuthCredentials(request);
+
+    if (!credentials) {
+      logger.debug("Returning 401 because credentials are missing");
+
+      return new Response("Unauthorized", {
+        status: 401,
+        headers: {
+          "WWW-Authenticate": 'Basic realm="Access to the registry"',
+        },
+      });
+    }
+
+    // Authenticate the request
+    const authenticatedEnv = await authenticateApiKey(credentials.password, {
+      allowPublicKey: false,
+    });
+
+    if (!authenticatedEnv) {
+      return new Response("Unauthorized", {
+        status: 401,
+      });
+    }
+
+    // construct a new url based on the url passed in and the registry url
+    const proxiedUrl = new URL(request.url);
+    proxiedUrl.host = this.host;
+
+    // Update the protocol to https if there is the x-forwarded-proto header
+    if (request.headers.get("x-forwarded-proto") === "https") {
+      proxiedUrl.protocol = "https:";
+    }
+
+    const updatedHeaders = this.#updateHeaders(request.headers);
+
+    const response = await fetch(proxiedUrl, {
+      method: request.method,
+      headers: updatedHeaders,
+      body: request.body,
+    });
+
+    const updatedResponseHeaders = this.#updateResponseHeaders(response.headers, request.url);
+
+    logger.debug("proxied request/response", {
+      proxiedUrl,
+      status: response.status,
+      statusText: response.statusText,
+      method: request.method,
+      requestHeaders: Object.fromEntries(updatedHeaders.entries()),
+      responseHeaders: Object.fromEntries(updatedResponseHeaders.entries()),
+    });
+
+    return new Response(response.body, {
+      status: response.status,
+      statusText: response.statusText,
+      headers: updatedResponseHeaders,
+    });
+  }
+
+  #getBasicAuthCredentials(request: Request) {
+    const authHeader = request.headers.get("authorization");
+
+    if (!authHeader) {
+      return;
+    }
+
+    const [type, credentials] = authHeader.split(" ");
+
+    if (type.toLowerCase() !== "basic") {
+      return;
+    }
+
+    const decoded = Buffer.from(credentials, "base64").toString("utf-8");
+    const [username, password] = decoded.split(":");
+
+    return { username, password };
+  }
+
+  // Updates the headers to be sent to the registry
+  // adds the docker auth
+  #updateHeaders(headers: Headers): Headers {
+    const newHeaders = new Headers(headers);
+
+    // Remove host, connection, accept-encoding, content-length, and authorization headers
+    newHeaders.delete("host");
+    newHeaders.delete("connection");
+    newHeaders.delete("accept-encoding");
+    newHeaders.delete("authorization");
+    newHeaders.delete("content-length");
+
+    newHeaders.set(
+      "authorization",
+      `Basic ${Buffer.from(`${this.auth.username}:${this.auth.password}`).toString("base64")}`
+    );
+
+    return newHeaders;
+  }
+
+  // Updates the headers to be sent back to the client
+  #updateResponseHeaders(headers: Headers, proxyUrl: string): Headers {
+    const newHeaders = new Headers(headers);
+
+    // Rewrite location headers to point to the proxy
+    if (headers.has("location")) {
+      const location = headers.get("location");
+
+      if (location) {
+        const proxiedLocation = new URL(location);
+        proxiedLocation.host = new URL(proxyUrl).host;
+
+        newHeaders.set("location", proxiedLocation.href);
+      }
+    }
+
+    return newHeaders;
+  }
+}
+
+export async function proxyToRegistry(request: Request) {
+  if (!env.DOCKER_REGISTRY_HOST || !env.DOCKER_REGISTRY_USERNAME || !env.DOCKER_REGISTRY_PASSWORD) {
+    return new Response(
+      "Could not proxy to the registry, please double check your DOCKER_REGISTRY_* env vars",
+      { status: 500 }
+    );
+  }
+
+  const registryProxy = new RegistryProxy(env.DOCKER_REGISTRY_HOST, {
+    username: env.DOCKER_REGISTRY_USERNAME,
+    password: env.DOCKER_REGISTRY_PASSWORD,
+  });
+
+  return await registryProxy.call(request);
+}

apps/webapp/app/services/tasks/performTaskOperation.server.ts

@@ -1,5 +1,3 @@
-import { env } from "process";
-import { Run } from "~/presenters/RunPresenter.server";
 import {
   FetchOperationSchema,
   FetchRequestInit,
@@ -11,13 +9,13 @@ import {
 import { RuntimeEnvironmentType, type Task } from "@trigger.dev/database";
 import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
 import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
+import { KitchenSinkTask, findKitchenSinkTask } from "~/models/task.server";
 import { formatUnknownError } from "~/utils/formatErrors.server";
 import { safeJsonFromResponse } from "~/utils/json";
+import { InitializeBackgroundTaskOperationService } from "../backgroundTasks/initializeBackgroundTaskOperation.server";
 import { logger } from "../logger.server";
 import { workerQueue } from "../worker.server";
 
-type FoundTask = Awaited<ReturnType<typeof findTask>>;
-
 export class PerformTaskOperationService {
   #prismaClient: PrismaClient;
 
@@ -26,7 +24,7 @@ export class PerformTaskOperationService {
   }
 
   public async call(id: string) {
-    const task = await findTask(this.#prismaClient, id);
+    const task = await findKitchenSinkTask(this.#prismaClient, id);
 
     if (!task) {
       return;
@@ -95,6 +93,11 @@ export class PerformTaskOperationService {
 
         return await this.#resumeTask(task, jsonBody);
       }
+      case "backgroundTask": {
+        const service = new InitializeBackgroundTaskOperationService();
+
+        return await service.call(task);
+      }
       default: {
         await this.#resumeTaskWithError(task, {
           message: `Unknown operation: ${task.operation}`,
@@ -104,7 +107,7 @@ export class PerformTaskOperationService {
   }
 
   #calculateRetryForResponse(
-    task: NonNullable<FoundTask>,
+    task: NonNullable<KitchenSinkTask>,
     retry: FetchRetryOptions | undefined,
     response: Response
   ): Date | undefined {
@@ -194,7 +197,7 @@ export class PerformTaskOperationService {
     });
   }
 
-  async #resumeTaskWithError(task: NonNullable<FoundTask>, output: any) {
+  async #resumeTaskWithError(task: NonNullable<KitchenSinkTask>, output: any) {
     await $transaction(this.#prismaClient, async (tx) => {
       await tx.task.update({
         where: { id: task.id },
@@ -220,7 +223,7 @@ export class PerformTaskOperationService {
     });
   }
 
-  async #resumeTask(task: NonNullable<FoundTask>, output: any) {
+  async #resumeTask(task: NonNullable<KitchenSinkTask>, output: any) {
     await $transaction(this.#prismaClient, async (tx) => {
       await tx.taskAttempt.updateMany({
         where: {
@@ -245,7 +248,7 @@ export class PerformTaskOperationService {
     });
   }
 
-  async #resumeRunExecution(task: NonNullable<FoundTask>, prisma: PrismaClientOrTransaction) {
+  async #resumeRunExecution(task: NonNullable<KitchenSinkTask>, prisma: PrismaClientOrTransaction) {
     await enqueueRunExecutionV2(task.run, prisma, {
       skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
     });
@@ -278,21 +281,6 @@ function hydrateRedactedString(value: RedactString): string {
   return result;
 }
 
-async function findTask(prisma: PrismaClient, id: string) {
-  return prisma.task.findUnique({
-    where: { id },
-    include: {
-      attempts: true,
-      run: {
-        include: {
-          environment: true,
-          queue: true,
-        },
-      },
-    },
-  });
-}
-
 // Add a random number of ms between 0ms and 5000ms
 function addJitterInMs() {
   return Math.floor(Math.random() * 5000);

apps/webapp/app/services/worker.server.ts

@@ -20,6 +20,9 @@ import { ActivateSourceService } from "./sources/activateSource.server";
 import { DeliverHttpSourceRequestService } from "./sources/deliverHttpSourceRequest.server";
 import { PerformTaskOperationService } from "./tasks/performTaskOperation.server";
 import { addMissingVersionField } from "@trigger.dev/core";
+import { ExecuteBackgroundTaskOperationService } from "./backgroundTasks/executeBackgroundTaskOperation.server";
+import { AutoScalePoolService } from "./backgroundTasks/autoScalePool.server";
+import { CreateExternalMachineService } from "./backgroundTasks/createExternalMachine.server";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -68,6 +71,15 @@ const workerCatalog = {
   connectionCreated: z.object({
     id: z.string(),
   }),
+  executeBackgroundTaskOperation: z.object({
+    id: z.string(),
+  }),
+  autoScalePool: z.object({
+    id: z.string(),
+  }),
+  createExternalMachine: z.object({
+    id: z.string(),
+  }),
 };
 
 const executionWorkerCatalog = {
@@ -285,6 +297,33 @@ function getWorkerQueue() {
           });
         },
       },
+      executeBackgroundTaskOperation: {
+        priority: 0, // smaller number = higher priority
+        maxAttempts: 5,
+        handler: async (payload, job) => {
+          const service = new ExecuteBackgroundTaskOperationService();
+
+          await service.call(payload.id);
+        },
+      },
+      autoScalePool: {
+        priority: 0, // smaller number = higher priority
+        maxAttempts: 5,
+        handler: async (payload, job) => {
+          const service = new AutoScalePoolService();
+
+          await service.call(payload.id);
+        },
+      },
+      createExternalMachine: {
+        priority: 0, // smaller number = higher priority
+        maxAttempts: 5,
+        handler: async (payload, job) => {
+          const service = new CreateExternalMachineService();
+
+          await service.call(payload.id);
+        },
+      },
     },
   });
 }

apps/webapp/app/zodfetch.server.ts

@@ -0,0 +1,108 @@
+import { z } from "zod";
+import { safeJsonParse } from "./utils/json";
+import { ErrorWithStackSchema } from "../../../packages/core/src";
+
+export type ZodResponse<TResponseSchema extends z.ZodTypeAny> =
+  | {
+      ok: true;
+      data: z.output<TResponseSchema>;
+      status: number;
+      headers: Headers;
+    }
+  | {
+      ok: false;
+      error: { message: string; name?: string; stack?: string };
+      status: number;
+      headers: Headers;
+    };
+
+const CommonErrorSchema = z.object({
+  error: z.string(),
+});
+
+export async function zodfetch<TResponseSchema extends z.ZodTypeAny>(
+  schema: TResponseSchema,
+  url: string,
+  requestInit?: RequestInit
+): Promise<ZodResponse<TResponseSchema>> {
+  const response = await fetch(url, requestInit);
+  const contentType = response.headers.get("content-type");
+
+  if (!response.ok) {
+    // Check to see if we have a JSON body
+    if (contentType?.includes("application/json")) {
+      const rawJsonBody = await response.text();
+      const jsonBody = safeJsonParse(rawJsonBody);
+
+      if (!jsonBody) {
+        return {
+          ok: false,
+          error: { message: "Failed to parse JSON response" },
+          status: response.status,
+          headers: response.headers,
+        };
+      }
+
+      const parsed = ErrorWithStackSchema.safeParse(jsonBody);
+
+      if (parsed.success) {
+        return {
+          ok: false,
+          error: parsed.data,
+          status: response.status,
+          headers: response.headers,
+        };
+      }
+
+      const commonParsed = CommonErrorSchema.safeParse(jsonBody);
+
+      if (commonParsed.success) {
+        return {
+          ok: false,
+          error: { message: commonParsed.data.error, name: response.statusText },
+          status: response.status,
+          headers: response.headers,
+        };
+      }
+
+      return {
+        ok: false,
+        error: jsonBody as any,
+        status: response.status,
+        headers: response.headers,
+      };
+    }
+
+    return {
+      ok: false,
+      error: { message: response.statusText },
+      status: response.status,
+      headers: response.headers,
+    };
+  }
+
+  const rawJsonBody = await response.text();
+  const jsonBody = safeJsonParse(rawJsonBody);
+
+  if (!jsonBody) {
+    return {
+      ok: true,
+      status: response.status,
+      headers: response.headers,
+      data: null as any,
+    };
+  }
+
+  const parsed = schema.safeParse(jsonBody);
+
+  if (!parsed.success) {
+    throw new Error(`Failed to parse response: ${parsed.error.message}`);
+  }
+
+  return {
+    ok: true,
+    data: parsed.data,
+    status: response.status,
+    headers: response.headers,
+  };
+}

apps/webapp/package.json

@@ -60,10 +60,12 @@
     "@remix-run/server-runtime": "1.19.2-pre.0",
     "@team-plain/typescript-sdk": "^2.2.0",
     "@trigger.dev/companyicons": "^1.5.14",
-    "@trigger.dev/database": "workspace:*",
     "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/database": "workspace:*",
     "@trigger.dev/sdk": "workspace:*",
     "@uiw/react-codemirror": "^4.19.5",
+    "archiver": "^6.0.1",
+    "async-retry": "^1.3.3",
     "class-variance-authority": "^0.5.2",
     "clsx": "^1.2.1",
     "compression": "^1.7.4",
@@ -133,6 +135,8 @@
     "@tailwindcss/typography": "^0.5.9",
     "@total-typescript/ts-reset": "^0.4.2",
     "@trigger.dev/tailwind-config": "workspace:*",
+    "@types/archiver": "^5.3.2",
+    "@types/async-retry": "^1.4.5",
     "@types/bcryptjs": "^2.4.2",
     "@types/compression": "^1.7.2",
     "@types/eslint": "^8.4.6",
@@ -148,6 +152,7 @@
     "@types/qs": "^6.9.7",
     "@types/react": "18.2.17",
     "@types/react-dom": "18.2.7",
+    "@types/retry": "^0.12.2",
     "@types/semver": "^7.3.13",
     "@types/simple-oauth2": "^5.0.4",
     "@types/slug": "^5.0.3",

examples/nextjs-background-tasks/src/tasks/task1.background.ts

@@ -5,7 +5,7 @@ import { z } from "zod";
 export default client.defineBackgroundTask({
   id: "task-1",
   name: "Task 1",
-  version: "1.0.0",
+  version: "1.0.2",
   schema: z.object({
     userName: z.string(),
   }),
@@ -19,6 +19,10 @@ export default client.defineBackgroundTask({
     // This code will run in the background, as it will be bundled and shipped to a trigger.dev background worker
     await new Promise((resolve) => setTimeout(resolve, 100000));
 
-    return `Task Response for user ${payload.userName}`;
+    return {
+      username: payload.userName,
+      foo: "bar",
+      message: `Task Response for user ${payload.userName}`,
+    };
   },
 });

packages/cli/package.json

@@ -37,6 +37,7 @@
   "devDependencies": {
     "@gmrchk/cli-testing-library": "^0.1.2",
     "@trigger.dev/tsconfig": "workspace:*",
+    "@types/dockerode": "^3.3.19",
     "@types/gradient-string": "^1.1.2",
     "@types/inquirer": "^9.0.3",
     "@types/jest": "^29.5.3",
@@ -44,6 +45,7 @@
     "@types/node": "16",
     "@types/node-fetch": "^2.6.2",
     "@types/semver": "^7.3.13",
+    "@types/tar": "^6.1.4",
     "jest": "^29.6.2",
     "rimraf": "^3.0.2",
     "ts-jest": "^29.1.1",
@@ -67,6 +69,7 @@
     "chokidar": "^3.5.3",
     "commander": "^9.4.1",
     "degit": "^2.8.4",
+    "dockerode": "^3.3.5",
     "dotenv": "^16.3.1",
     "esbuild": "^0.19.2",
     "execa": "^7.0.0",
@@ -85,6 +88,7 @@
     "semver": "^7.5.0",
     "simple-git": "^3.19.0",
     "spawn-please": "^2.0.2",
+    "tar": "^6.2.0",
     "terminal-link": "^3.0.0",
     "tsconfck": "^2.1.2",
     "url": "^0.11.1",

packages/cli/src/commands/deploy.ts

@@ -6,10 +6,12 @@ import { build } from "esbuild";
 import pathModule from "node:path";
 import util from "util";
 import { z } from "zod";
+import { DockerWrapper } from "../utils/docker";
+import { downloadArchiveToTempDir } from "../utils/extractArchiveToTempDir";
+import { getTriggerApiDetails } from "../utils/getTriggerApiDetails";
 import { logger } from "../utils/logger";
 import { listPackageDependencies } from "../utils/packageManagers";
 import { resolvePath } from "../utils/parseNameAndPath";
-import { getTriggerApiDetails } from "../utils/getTriggerApiDetails";
 import { TriggerApi } from "../utils/triggerApi";
 
 const asyncExecFile = util.promisify(childProcess.execFile);
@@ -21,6 +23,14 @@ export const DeployCommandOptionsSchema = z.object({
 
 export type DevCommandOptions = z.infer<typeof DeployCommandOptionsSchema>;
 
+type DeployedArtifact = {
+  id: string;
+  hash: string;
+  image: string;
+  tag: string;
+  task: DeployBackgroundTaskRequestBody;
+};
+
 export async function deployCommand(path: string, anyOptions: any) {
   const result = DeployCommandOptionsSchema.safeParse(anyOptions);
 
@@ -88,6 +98,8 @@ export async function deployCommand(path: string, anyOptions: any) {
 
   const apiClient = new TriggerApi(apiDetails.apiKey, apiDetails.apiUrl);
 
+  const artifacts: Array<DeployedArtifact> = [];
+
   for (const task of tasks) {
     const response = await apiClient.deployBackgroundTask(task);
 
@@ -95,8 +107,80 @@ export async function deployCommand(path: string, anyOptions: any) {
       logger.error(`Failed to deploy ${task.id}@${task.version}: ${response.error}`);
     } else {
       logger.info(`Deployed ${task.id}@${task.version}#${response.data.hash}`);
+
+      artifacts.push({
+        id: response.data.id,
+        hash: response.data.hash,
+        image: response.data.image,
+        tag: response.data.tag,
+        task,
+      });
     }
   }
+
+  // For each artifact, we need to build a docker image and push it to the registry
+  // The registry host is the TRIGGER_API_URL
+  for (const artifact of artifacts) {
+    const results = await buildAndPushDockerImage(apiDetails.apiUrl, apiDetails.apiKey, artifact);
+
+    if (results) {
+      await apiClient.createBackgroundTaskArtifactImage(artifact.id, {
+        digest: results.digest,
+        name: artifact.image,
+        tag: artifact.tag,
+        size: results.size,
+      });
+    }
+  }
+}
+
+async function buildAndPushDockerImage(
+  registryUrl: string,
+  apiKey: string,
+  artifact: DeployedArtifact
+) {
+  const registry = new URL(registryUrl).host;
+  const imageName = `${registry}/${artifact.image}:${artifact.tag}`;
+
+  const docker = new DockerWrapper({ docker: { socketPath: "/var/run/docker.sock" } });
+
+  logger.info(
+    `Building docker image ${imageName} for task ${artifact.task.id}@${artifact.task.version}...`
+  );
+
+  const artifactPath = await downloadArtifact(registryUrl, artifact);
+
+  const buildResults = await docker.buildImage(artifactPath, {
+    t: imageName,
+    dockerfile: "ctx/Dockerfile",
+    platform: "linux/amd64",
+    nocache: true,
+  });
+
+  if (!buildResults) {
+    logger.error(`Failed to build docker image ${imageName}`);
+    return;
+  }
+
+  logger.info(`Built docker image ${imageName}: ${JSON.stringify(buildResults)}`);
+
+  const pushResults = await docker.pushImage(imageName, {
+    authconfig: {
+      username: "x",
+      password: apiKey,
+      serveraddress: "https://index.docker.io/v1",
+    },
+  });
+
+  logger.info(`Pushed docker image ${imageName}: ${JSON.stringify(pushResults)}`);
+
+  return pushResults;
+}
+
+async function downloadArtifact(registryUrl: string, artifact: DeployedArtifact): Promise<string> {
+  const artifactArchiveUrl = `${registryUrl}/api/v1/background/artifacts/${artifact.id}.tgz`;
+
+  return await downloadArchiveToTempDir(artifactArchiveUrl, artifact.id);
 }
 
 async function gatherBackgroundTaskDeployment(

packages/cli/src/utils/docker.ts

@@ -0,0 +1,130 @@
+import Docker, {
+  DockerOptions,
+  ImageBuildContext,
+  ImageBuildOptions,
+  ImagePushOptions,
+} from "dockerode";
+import { z } from "zod";
+
+const BuildAuxResultSchema = z.object({
+  aux: z.object({
+    ID: z.string(),
+  }),
+});
+
+const BuildStreamResultSchema = z.object({
+  stream: z.string(),
+});
+
+const PushAuxResultSchema = z.object({
+  aux: z.object({
+    Tag: z.string(),
+    Digest: z.string(),
+    Size: z.number(),
+  }),
+});
+
+export type DockerWrapperOptions = {
+  docker?: DockerOptions;
+};
+
+export class DockerWrapper {
+  readonly #docker: Docker;
+
+  constructor(options?: DockerWrapperOptions) {
+    this.#docker = new Docker(options?.docker);
+  }
+
+  // https://jsonhero.io/j/PzXccoO3lCSg
+  async buildImage(
+    context: ImageBuildContext | string,
+    options: ImageBuildOptions
+  ): Promise<{ digest: string; image: string } | undefined> {
+    const stream = await this.#docker.buildImage(context, options);
+
+    const buildResults = await new Promise<any[]>((resolve, reject) => {
+      this.#docker.modem.followProgress(stream, (err, res) => (err ? reject(err) : resolve(res)));
+    });
+
+    return this.#parseBuildResults(buildResults);
+  }
+
+  // https://jsonhero.io/j/BmJcRaX5rx1s
+  async pushImage(
+    imageName: string,
+    options?: ImagePushOptions
+  ): Promise<{ tag: string; digest: string; size: number } | undefined> {
+    const image = this.#docker.getImage(imageName);
+
+    const stream = await image.push(options);
+
+    const pushResults = await new Promise((resolve, reject) => {
+      this.#docker.modem.followProgress(stream, (err, res) => (err ? reject(err) : resolve(res)));
+    });
+
+    return this.#parsePushResults(pushResults);
+  }
+
+  #parsePushResults(
+    pushResults: unknown
+  ): { tag: string; digest: string; size: number } | undefined {
+    const resultsArray = z.array(z.any()).safeParse(pushResults);
+
+    if (!resultsArray.success) {
+      return;
+    }
+
+    const auxResult = findWithSchema(PushAuxResultSchema, resultsArray.data);
+
+    if (!auxResult) {
+      return undefined;
+    }
+
+    return {
+      tag: auxResult.aux.Tag,
+      digest: auxResult.aux.Digest,
+      size: auxResult.aux.Size,
+    };
+  }
+
+  #parseBuildResults(buildResults: any[]): { digest: string; image: string } | undefined {
+    const auxResult = findWithSchema(BuildAuxResultSchema, buildResults);
+
+    if (!auxResult) {
+      return undefined;
+    }
+
+    const lastResult = buildResults[buildResults.length - 1];
+    const parsedLastResult = BuildStreamResultSchema.safeParse(lastResult);
+
+    if (!parsedLastResult.success) {
+      return undefined;
+    }
+
+    const image = this.#parseImageFromStream(parsedLastResult.data.stream);
+
+    if (!image) {
+      return undefined;
+    }
+
+    return {
+      digest: auxResult.aux.ID,
+      image,
+    };
+  }
+
+  // Successfully tagged eric-webapp.trigger.dev/clm7ndmb60009dyeoe0edbf32-task-1:1.0.1
+  #parseImageFromStream(stream: string): string | undefined {
+    const match = stream.match(/Successfully tagged ([^:]+):([^ ]+)/);
+
+    if (!match) {
+      return undefined;
+    }
+
+    return match[1]?.trim();
+  }
+}
+
+function findWithSchema<T>(schema: z.Schema<T>, items: any[]): T | undefined {
+  return items.find((item) => (schema.safeParse(item).success ? true : false));
+}

packages/cli/src/utils/extractArchiveToTempDir.ts

@@ -0,0 +1,54 @@
+import { createWriteStream } from "node:fs";
+import { createTempDir } from "./fileSystem";
+import { pipeline } from "node:stream";
+import { promisify } from "node:util";
+import tar from "tar";
+
+const streamPipeline = promisify(pipeline);
+
+// A node.js function to download and extract a tarball to a temporary directory, and return the path to the directory.
+export async function extractArchiveToTempDir(url: string, prefix: string): Promise<string> {
+  const archiveTempDir = await createTempDir(`${prefix}-archive`);
+  const tempDir = await createTempDir(prefix);
+  const tempFilePath = `${archiveTempDir}/file.tgz`;
+
+  // Download the .tgz file
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download .tgz file (${response.status} ${response.statusText})`);
+  }
+
+  if (!response.body) {
+    throw new Error("Failed to download .tgz file (no body)");
+  }
+
+  const fileStream = createWriteStream(tempFilePath);
+  // @ts-ignore
+  await streamPipeline(response.body, fileStream);
+
+  // Extract the contents to the temporary directory
+  await tar.x({ C: tempDir, file: tempFilePath });
+
+  return tempDir;
+}
+
+export async function downloadArchiveToTempDir(url: string, prefix: string): Promise<string> {
+  const archiveTempDir = await createTempDir(`${prefix}-archive`);
+  const tempFilePath = `${archiveTempDir}/file.tgz`;
+
+  // Download the .tgz file
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download .tgz file (${response.status} ${response.statusText})`);
+  }
+
+  if (!response.body) {
+    throw new Error("Failed to download .tgz file (no body)");
+  }
+
+  const fileStream = createWriteStream(tempFilePath);
+  // @ts-ignore
+  await streamPipeline(response.body, fileStream);
+
+  return tempFilePath;
+}

packages/cli/src/utils/fileSystem.ts

@@ -1,6 +1,7 @@
-import fsModule, { writeFile } from "fs/promises";
-import fsSync from "fs";
-import pathModule from "path";
+import fsModule, { mkdtemp, writeFile } from "node:fs/promises";
+import fsSync from "node:fs";
+import pathModule from "node:path";
+import os from "node:os";
 
 // Creates a file at the given path, if the directory doesn't exist it will be created
 export async function createFile(path: string, contents: string): Promise<string> {
@@ -43,3 +44,33 @@ export function readJSONFileSync(path: string) {
 
   return JSON.parse(fileContents);
 }
+
+export async function createTempDir(prefix: string) {
+  return await mkdtemp(pathModule.join(os.tmpdir(), `${prefix}-`));
+}
+
+// Recursively list all the files in the directory, and returns an array a relative paths
+export async function listFilesInDir(dir: string): Promise<string[]> {
+  async function listFilesInDirRecursive(dir: string): Promise<string[]> {
+    const files = await fsModule.readdir(dir);
+
+    const filePaths = await Promise.all(
+      files.map(async (file) => {
+        const filePath = pathModule.join(dir, file);
+        const stat = await fsModule.stat(filePath);
+
+        if (stat.isDirectory()) {
+          return await listFilesInDirRecursive(filePath);
+        }
+
+        return filePath;
+      })
+    );
+
+    return filePaths.flat();
+  }
+
+  const allFiles = await listFilesInDirRecursive(dir);
+
+  return allFiles.map((file) => pathModule.relative(dir, file));
+}

packages/cli/src/utils/triggerApi.ts

@@ -1,4 +1,6 @@
 import {
+  CreateBackgroundTaskImageRequestBody,
+  CreateBackgroundTaskImageResponseBody,
   DeployBackgroundTaskRequestBody,
   DeployBackgroundTaskResponseBody,
 } from "@trigger.dev/core";
@@ -227,6 +229,65 @@ export class TriggerApi {
       data: data as any as DeployBackgroundTaskResponseBody,
     };
   }
+
+  async createBackgroundTaskArtifactImage(
+    id: string,
+    options: CreateBackgroundTaskImageRequestBody
+  ): Promise<ApiResponse<CreateBackgroundTaskImageResponseBody>> {
+    const response = await fetch(`${this.baseUrl}/api/v1/background/artifacts/${id}/images`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify(options),
+    });
+
+    if (!response.ok) {
+      const rawBody = await response.text();
+
+      if (typeof rawBody === "string") {
+        const rawJson = safeJsonParse(rawBody);
+
+        if (!rawJson) {
+          return {
+            ok: false,
+            error: "An unknown issue occurred creating image on Trigger.dev",
+            retryable: true,
+          };
+        }
+
+        const parsedJson = z.object({ error: z.string() }).safeParse(rawJson);
+
+        if (!parsedJson.success) {
+          return {
+            ok: false,
+            error: "An unknown issue occurred creating image on Trigger.dev",
+            retryable: true,
+          };
+        }
+
+        return {
+          ok: false,
+          error: parsedJson.data.error,
+          retryable: RETRYABLE_PATTERN.test(parsedJson.data.error),
+        };
+      } else {
+        return {
+          ok: false,
+          error: "An unknown issue occurred creating image on Trigger.dev",
+          retryable: true,
+        };
+      }
+    }
+
+    const data = await response.json();
+
+    return {
+      ok: true,
+      data: data as any as CreateBackgroundTaskImageResponseBody,
+    };
+  }
 }
 
 function safeJsonParse(raw: string | null | undefined): unknown {

packages/core/src/logger.ts

@@ -30,6 +30,15 @@ export class Logger {
     this.#jsonReplacer = jsonReplacer;
   }
 
+  child(name: string): Logger {
+    return new Logger(
+      `${this.#name}:${name}`,
+      logLevels[this.#level],
+      this.#filteredKeys,
+      this.#jsonReplacer
+    );
+  }
+
   // Return a new Logger instance with the same name and a new log level
   // but filter out the keys from the log messages (at any level)
   filter(...keys: string[]) {

packages/core/src/schemas/api.ts

@@ -288,7 +288,9 @@ export const BackgroundTaskMetadataSchema = z.object({
   enabled: z.boolean(),
   cpu: z.number(),
   memory: z.number(),
+  diskSizeInGB: z.number().optional(),
   concurrency: z.number(),
+  region: z.string().optional(),
   secrets: z.record(z.string()).optional(),
 });
 
@@ -607,7 +609,7 @@ export const RunTaskOptionsSchema = z.object({
   /** Allows you to link the Integration connection in the logs. This is handled automatically in integrations.  */
   connectionKey: z.string().optional(),
   /** An operation you want to perform on the Trigger.dev platform, current only "fetch" is supported. If you wish to `fetch` use [`io.backgroundFetch()`](https://trigger.dev/docs/sdk/io/backgroundfetch) instead. */
-  operation: z.enum(["fetch"]).optional(),
+  operation: z.enum(["fetch", "backgroundTask"]).optional(),
   /** A No Operation means that the code won't be executed. This is used internally to implement features like [io.wait()](https://trigger.dev/docs/sdk/io/wait).  */
   noop: z.boolean().default(false),
   redact: RedactSchema.optional(),
@@ -767,6 +769,8 @@ export type DeployBackgroundTaskRequestBody = z.infer<typeof DeployBackgroundTas
 export const DeployBackgroundTaskResponseBodySchema = z.object({
   id: z.string(),
   hash: z.string(),
+  image: z.string(),
+  tag: z.string(),
   createdAt: z.coerce.date(),
   updatedAt: z.coerce.date(),
 });
@@ -774,3 +778,31 @@ export const DeployBackgroundTaskResponseBodySchema = z.object({
 export type DeployBackgroundTaskResponseBody = z.infer<
   typeof DeployBackgroundTaskResponseBodySchema
 >;
+
+export const CreateBackgroundTaskImageRequestBodySchema = z.object({
+  name: z.string(),
+  tag: z.string(),
+  digest: z.string(),
+  size: z.number(),
+});
+
+export type CreateBackgroundTaskImageRequestBody = z.infer<
+  typeof CreateBackgroundTaskImageRequestBodySchema
+>;
+
+export const CreateBackgroundTaskImageResponseBodySchema = z.object({
+  id: z.string(),
+  backgroundTaskArtifactId: z.string(),
+  backgroundTaskId: z.string(),
+  name: z.string(),
+  tag: z.string(),
+  digest: z.string(),
+  size: z.number(),
+  provider: z.string(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+});
+
+export type CreateBackgroundTaskImageResponseBody = z.infer<
+  typeof CreateBackgroundTaskImageResponseBodySchema
+>;

packages/core/src/schemas/backgroundTasks.ts

@@ -0,0 +1,7 @@
+import { z } from "zod";
+
+export const BackgroundTaskOperationParamsSchema = z.object({
+  id: z.string(),
+  version: z.string(),
+  payload: z.any(),
+});

packages/core/src/schemas/index.ts

@@ -12,3 +12,4 @@ export * from "./fetch";
 export * from "./events";
 export * from "./runs";
 export * from "./addMissingVersionField";
+export * from "./backgroundTasks";

packages/database/prisma/migrations/20230912105319_add_more_background_task_models/migration.sql

@@ -0,0 +1,78 @@
+-- CreateEnum
+CREATE TYPE "BackgroundTaskProvider" AS ENUM ('FLY_IO');
+
+-- CreateEnum
+CREATE TYPE "BackgroundTaskMachineStatus" AS ENUM ('CREATED', 'STARTING', 'STARTED', 'STOPPING', 'STOPPED', 'DESTROYING', 'DESTROYED', 'REPLACING');
+
+-- CreateEnum
+CREATE TYPE "BackgroundTaskOperationStatus" AS ENUM ('PENDING', 'STARTED', 'SUCCESS', 'FAILURE');
+
+-- CreateTable
+CREATE TABLE "BackgroundTaskImage" (
+    "id" TEXT NOT NULL,
+    "provider" "BackgroundTaskProvider" NOT NULL,
+    "name" TEXT NOT NULL,
+    "tag" TEXT NOT NULL,
+    "digest" TEXT NOT NULL,
+    "size" INTEGER NOT NULL,
+    "backgroundTaskId" TEXT NOT NULL,
+    "backgroundTaskArtifactId" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "BackgroundTaskImage_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "BackgroundTaskMachine" (
+    "id" TEXT NOT NULL,
+    "externalId" TEXT NOT NULL,
+    "provider" "BackgroundTaskProvider" NOT NULL,
+    "data" JSONB NOT NULL,
+    "status" "BackgroundTaskMachineStatus" NOT NULL DEFAULT 'CREATED',
+    "backgroundTaskId" TEXT NOT NULL,
+    "backgroundTaskVersionId" TEXT NOT NULL,
+    "backgroundTaskImageId" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "BackgroundTaskMachine_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "BackgroundTaskOperation" (
+    "id" TEXT NOT NULL,
+    "backgroundTaskId" TEXT NOT NULL,
+    "backgroundTaskVersionId" TEXT NOT NULL,
+    "payload" JSONB NOT NULL,
+    "output" JSONB,
+    "error" JSONB,
+    "status" "BackgroundTaskOperationStatus" NOT NULL DEFAULT 'PENDING',
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "startedAt" TIMESTAMP(3),
+    "endedAt" TIMESTAMP(3),
+
+    CONSTRAINT "BackgroundTaskOperation_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskImage" ADD CONSTRAINT "BackgroundTaskImage_backgroundTaskId_fkey" FOREIGN KEY ("backgroundTaskId") REFERENCES "BackgroundTask"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskImage" ADD CONSTRAINT "BackgroundTaskImage_backgroundTaskArtifactId_fkey" FOREIGN KEY ("backgroundTaskArtifactId") REFERENCES "BackgroundTaskArtifact"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskMachine" ADD CONSTRAINT "BackgroundTaskMachine_backgroundTaskId_fkey" FOREIGN KEY ("backgroundTaskId") REFERENCES "BackgroundTask"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskMachine" ADD CONSTRAINT "BackgroundTaskMachine_backgroundTaskVersionId_fkey" FOREIGN KEY ("backgroundTaskVersionId") REFERENCES "BackgroundTaskVersion"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskMachine" ADD CONSTRAINT "BackgroundTaskMachine_backgroundTaskImageId_fkey" FOREIGN KEY ("backgroundTaskImageId") REFERENCES "BackgroundTaskImage"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskOperation" ADD CONSTRAINT "BackgroundTaskOperation_backgroundTaskId_fkey" FOREIGN KEY ("backgroundTaskId") REFERENCES "BackgroundTask"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskOperation" ADD CONSTRAINT "BackgroundTaskOperation_backgroundTaskVersionId_fkey" FOREIGN KEY ("backgroundTaskVersionId") REFERENCES "BackgroundTaskVersion"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/database/prisma/migrations/20230912122347_add_unique_constraint_on_background_task_images/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[backgroundTaskArtifactId,digest]` on the table `BackgroundTaskImage` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- CreateIndex
+CREATE UNIQUE INDEX "BackgroundTaskImage_backgroundTaskArtifactId_digest_key" ON "BackgroundTaskImage"("backgroundTaskArtifactId", "digest");

packages/database/prisma/migrations/20230912122608_rename_background_task_provider_enum/migration.sql

@@ -0,0 +1,20 @@
+/*
+  Warnings:
+
+  - Changed the type of `provider` on the `BackgroundTaskImage` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required.
+  - Changed the type of `provider` on the `BackgroundTaskMachine` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required.
+
+*/
+-- CreateEnum
+CREATE TYPE "BackgroundTaskProviderStrategy" AS ENUM ('FLY_IO');
+
+-- AlterTable
+ALTER TABLE "BackgroundTaskImage" DROP COLUMN "provider",
+ADD COLUMN     "provider" "BackgroundTaskProviderStrategy" NOT NULL;
+
+-- AlterTable
+ALTER TABLE "BackgroundTaskMachine" DROP COLUMN "provider",
+ADD COLUMN     "provider" "BackgroundTaskProviderStrategy" NOT NULL;
+
+-- DropEnum
+DROP TYPE "BackgroundTaskProvider";

packages/database/prisma/migrations/20230912122727_add_unsupported_provider_strategy/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "BackgroundTaskProviderStrategy" ADD VALUE 'UNSUPPORTED';

packages/database/prisma/migrations/20230912130758_add_task_to_background_task_operation/migration.sql

@@ -0,0 +1,15 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[taskId]` on the table `BackgroundTaskOperation` will be added. If there are existing duplicate values, this will fail.
+  - Added the required column `taskId` to the `BackgroundTaskOperation` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "BackgroundTaskOperation" ADD COLUMN     "taskId" TEXT NOT NULL;
+
+-- CreateIndex
+CREATE UNIQUE INDEX "BackgroundTaskOperation_taskId_key" ON "BackgroundTaskOperation"("taskId");
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskOperation" ADD CONSTRAINT "BackgroundTaskOperation_taskId_fkey" FOREIGN KEY ("taskId") REFERENCES "Task"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/database/prisma/migrations/20230912132333_add_waiting_on_image_to_operation_status/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "BackgroundTaskOperationStatus" ADD VALUE 'WAITING_ON_IMAGE';

packages/database/prisma/migrations/20230912132627_adding_pending_to_machine_status/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "BackgroundTaskMachineStatus" ADD VALUE 'PENDING';

packages/database/prisma/migrations/20230912133614_added_machine_pools/migration.sql

@@ -0,0 +1,45 @@
+/*
+  Warnings:
+
+  - Added the required column `poolId` to the `BackgroundTaskMachine` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterEnum
+ALTER TYPE "BackgroundTaskOperationStatus" ADD VALUE 'ASSIGNED_TO_POOL';
+
+-- AlterTable
+ALTER TABLE "BackgroundTaskMachine" ADD COLUMN     "poolId" TEXT NOT NULL;
+
+-- AlterTable
+ALTER TABLE "BackgroundTaskOperation" ADD COLUMN     "poolId" TEXT;
+
+-- CreateTable
+CREATE TABLE "BackgroundTaskMachinePool" (
+    "id" TEXT NOT NULL,
+    "provider" "BackgroundTaskProviderStrategy" NOT NULL,
+    "imageId" TEXT NOT NULL,
+    "backgroundTaskVersionId" TEXT NOT NULL,
+    "backgroundTaskId" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "BackgroundTaskMachinePool_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "BackgroundTaskMachinePool_backgroundTaskVersionId_imageId_key" ON "BackgroundTaskMachinePool"("backgroundTaskVersionId", "imageId");
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskMachine" ADD CONSTRAINT "BackgroundTaskMachine_poolId_fkey" FOREIGN KEY ("poolId") REFERENCES "BackgroundTaskMachinePool"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskMachinePool" ADD CONSTRAINT "BackgroundTaskMachinePool_imageId_fkey" FOREIGN KEY ("imageId") REFERENCES "BackgroundTaskImage"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskMachinePool" ADD CONSTRAINT "BackgroundTaskMachinePool_backgroundTaskVersionId_fkey" FOREIGN KEY ("backgroundTaskVersionId") REFERENCES "BackgroundTaskVersion"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskMachinePool" ADD CONSTRAINT "BackgroundTaskMachinePool_backgroundTaskId_fkey" FOREIGN KEY ("backgroundTaskId") REFERENCES "BackgroundTask"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "BackgroundTaskOperation" ADD CONSTRAINT "BackgroundTaskOperation_poolId_fkey" FOREIGN KEY ("poolId") REFERENCES "BackgroundTaskMachinePool"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/database/prisma/migrations/20230912142718_make_external_data_optional_on_task_machines/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "BackgroundTaskMachine" ALTER COLUMN "externalId" DROP NOT NULL,
+ALTER COLUMN "data" DROP NOT NULL;

packages/database/prisma/migrations/20230912152725_change_machine_status_default/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "BackgroundTaskMachine" ALTER COLUMN "status" SET DEFAULT 'PENDING';

packages/database/prisma/migrations/20230913115949_add_region_to_background_task_version/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "BackgroundTaskVersion" ADD COLUMN     "region" TEXT NOT NULL DEFAULT 'iad';

packages/database/prisma/migrations/20230913120657_add_region_to_pools/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `region` to the `BackgroundTaskMachinePool` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "BackgroundTaskMachinePool" ADD COLUMN     "region" TEXT NOT NULL;

packages/database/prisma/migrations/20230913120826_add_other_config_to_pools/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "BackgroundTaskMachinePool" ADD COLUMN     "concurrency" INTEGER NOT NULL DEFAULT 1,
+ADD COLUMN     "cpu" INTEGER NOT NULL DEFAULT 1,
+ADD COLUMN     "memory" INTEGER NOT NULL DEFAULT 256;

packages/database/prisma/migrations/20230913121002_add_disk_size_to_models/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "BackgroundTaskMachinePool" ADD COLUMN     "diskSize" INTEGER NOT NULL DEFAULT 1;
+
+-- AlterTable
+ALTER TABLE "BackgroundTaskVersion" ADD COLUMN     "diskSize" INTEGER NOT NULL DEFAULT 1;

packages/database/prisma/schema.prisma

@@ -817,9 +817,10 @@ model Task {
   runConnection   RunConnection? @relation(fields: [runConnectionId], references: [id], onDelete: Cascade, onUpdate: Cascade)
   runConnectionId String?
 
-  children   Task[]            @relation("TaskParent")
-  executions JobRunExecution[]
-  attempts   TaskAttempt[]
+  children                Task[]                   @relation("TaskParent")
+  executions              JobRunExecution[]
+  attempts                TaskAttempt[]
+  backgroundTaskOperation BackgroundTaskOperation?
 
   @@unique([runId, idempotencyKey])
 }
@@ -1101,6 +1102,11 @@ model BackgroundTask {
 
   deletedAt DateTime?
 
+  operations                BackgroundTaskOperation[]
+  machines                  BackgroundTaskMachine[]
+  images                    BackgroundTaskImage[]
+  BackgroundTaskMachinePool BackgroundTaskMachinePool[]
+
   @@unique([projectId, slug])
 }
 
@@ -1108,9 +1114,11 @@ model BackgroundTaskVersion {
   id      String @id @default(cuid())
   version String
 
-  cpu         Int @default(1)
-  memory      Int @default(256)
-  concurrency Int @default(1)
+  cpu         Int    @default(1)
+  memory      Int    @default(256)
+  concurrency Int    @default(1)
+  diskSize    Int    @default(1)
+  region      String @default("iad")
 
   backgroundTask   BackgroundTask @relation(fields: [backgroundTaskId], references: [id], onDelete: Cascade, onUpdate: Cascade)
   backgroundTaskId String
@@ -1130,8 +1138,11 @@ model BackgroundTaskVersion {
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
-  aliases BackgroundTaskAlias[]
-  secrets BackgroundTaskSecret[]
+  aliases    BackgroundTaskAlias[]
+  secrets    BackgroundTaskSecret[]
+  operations BackgroundTaskOperation[]
+  machines   BackgroundTaskMachine[]
+  pools      BackgroundTaskMachinePool[]
 
   @@unique([backgroundTaskId, version, environmentId])
 }
@@ -1186,5 +1197,136 @@ model BackgroundTaskArtifact {
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
+  images BackgroundTaskImage[]
+
   @@unique([backgroundTaskId, version, hash])
 }
+
+enum BackgroundTaskProviderStrategy {
+  FLY_IO
+  UNSUPPORTED
+}
+
+model BackgroundTaskImage {
+  id       String                         @id @default(cuid())
+  provider BackgroundTaskProviderStrategy
+  name     String
+  tag      String
+  digest   String
+  size     Int
+
+  backgroundTask   BackgroundTask @relation(fields: [backgroundTaskId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskId String
+
+  backgroundTaskArtifact   BackgroundTaskArtifact @relation(fields: [backgroundTaskArtifactId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskArtifactId String
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  machines BackgroundTaskMachine[]
+  pools    BackgroundTaskMachinePool[]
+
+  @@unique([backgroundTaskArtifactId, digest])
+}
+
+model BackgroundTaskMachine {
+  id String @id @default(cuid())
+
+  externalId String?
+  provider   BackgroundTaskProviderStrategy
+  data       Json?
+  status     BackgroundTaskMachineStatus    @default(PENDING)
+
+  backgroundTask   BackgroundTask @relation(fields: [backgroundTaskId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskId String
+
+  backgroundTaskVersion   BackgroundTaskVersion @relation(fields: [backgroundTaskVersionId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskVersionId String
+
+  backgroundTaskImage   BackgroundTaskImage @relation(fields: [backgroundTaskImageId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskImageId String
+
+  pool   BackgroundTaskMachinePool @relation(fields: [poolId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  poolId String
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+}
+
+enum BackgroundTaskMachineStatus {
+  PENDING
+  CREATED
+  STARTING
+  STARTED
+  STOPPING
+  STOPPED
+  DESTROYING
+  DESTROYED
+  REPLACING
+}
+
+model BackgroundTaskMachinePool {
+  id String @id @default(cuid())
+
+  provider BackgroundTaskProviderStrategy
+
+  region      String
+  cpu         Int    @default(1)
+  memory      Int    @default(256)
+  concurrency Int    @default(1)
+  diskSize    Int    @default(1)
+
+  image   BackgroundTaskImage @relation(fields: [imageId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  imageId String
+
+  backgroundTaskVersion   BackgroundTaskVersion @relation(fields: [backgroundTaskVersionId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskVersionId String
+
+  backgroundTask   BackgroundTask @relation(fields: [backgroundTaskId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskId String
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  machines   BackgroundTaskMachine[]
+  operations BackgroundTaskOperation[]
+
+  @@unique([backgroundTaskVersionId, imageId])
+}
+
+model BackgroundTaskOperation {
+  id String @id @default(cuid())
+
+  backgroundTask   BackgroundTask @relation(fields: [backgroundTaskId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskId String
+
+  backgroundTaskVersion   BackgroundTaskVersion @relation(fields: [backgroundTaskVersionId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundTaskVersionId String
+
+  task   Task   @relation(fields: [taskId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  taskId String @unique
+
+  pool   BackgroundTaskMachinePool? @relation(fields: [poolId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  poolId String?
+
+  payload Json
+  output  Json?
+  error   Json?
+
+  status BackgroundTaskOperationStatus @default(PENDING)
+
+  createdAt DateTime  @default(now())
+  updatedAt DateTime  @updatedAt
+  startedAt DateTime?
+  endedAt   DateTime?
+}
+
+enum BackgroundTaskOperationStatus {
+  PENDING
+  WAITING_ON_IMAGE
+  ASSIGNED_TO_POOL
+  STARTED
+  SUCCESS
+  FAILURE
+}

packages/trigger-sdk/package.json

@@ -20,6 +20,7 @@
   },
   "scripts": {
     "clean": "rimraf dist",
+    "dev": "tsup --watch --dts-resolve",
     "build": "npm run clean && npm run build:tsup",
     "build:tsup": "tsup --dts-resolve",
     "typecheck": "tsc --noEmit"

packages/trigger-sdk/src/backgroundTask.ts

@@ -2,8 +2,9 @@ import { BackgroundTaskMetadata, LogLevel } from "@trigger.dev/core";
 import { z } from "zod";
 import { TriggerClient } from "./triggerClient";
 import { slugifyId } from "./utils";
+import { runLocalStorage } from "./runLocalStorage";
 
-export type BackgroundTaskOptions<TPayload = any> = {
+export type BackgroundTaskOptions<TPayload = any, TRunResult = any> = {
   id: string;
   name: string;
   version: string;
@@ -13,21 +14,23 @@ export type BackgroundTaskOptions<TPayload = any> = {
   cpu?: 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128;
   memory?: 256 | 512 | 1024 | 2048 | 4096 | 8192 | 16384 | 32768;
   concurrency?: number;
+  region?: string;
+  diskSizeInGB?: number;
 
   secrets?: {
     [key: string]: string;
   };
 
   enabled?: boolean;
-  run: (payload: TPayload) => Promise<any>;
+  run: (payload: TPayload) => Promise<TRunResult>;
 };
 
-export class BackgroundTask<TPayload = any> {
-  readonly options: BackgroundTaskOptions<TPayload>;
+export class BackgroundTask<TPayload = any, TRunResult = any> {
+  readonly options: BackgroundTaskOptions<TPayload, TRunResult>;
 
   client: TriggerClient;
 
-  constructor(client: TriggerClient, options: BackgroundTaskOptions<TPayload>) {
+  constructor(client: TriggerClient, options: BackgroundTaskOptions<TPayload, TRunResult>) {
     this.client = client;
     this.options = options;
     this.#validate();
@@ -59,7 +62,21 @@ export class BackgroundTask<TPayload = any> {
     return this.options.logLevel;
   }
 
-  public async invoke(key: string, payload: TPayload): Promise<any> {}
+  public async invoke(key: string | string[], payload: TPayload): Promise<TRunResult> {
+    if (!this.enabled) {
+      throw new Error(`Cannot invoke a disabled background task: ${this.id}`);
+    }
+
+    const runStore = runLocalStorage.getStore();
+
+    if (!runStore) {
+      throw new Error("Cannot invoke a background task outside of a job run");
+    }
+
+    const { io, ctx } = runStore;
+
+    return await io.backgroundTask(key, this.id, this.version, payload);
+  }
 
   toJSON(): BackgroundTaskMetadata {
     return {
@@ -69,7 +86,9 @@ export class BackgroundTask<TPayload = any> {
       enabled: this.enabled,
       cpu: this.options.cpu ?? 1,
       memory: this.options.memory ?? 256,
+      region: this.options.region,
       concurrency: this.options.concurrency ?? 1,
+      diskSizeInGB: this.options.diskSizeInGB,
       secrets: this.options.secrets ?? {},
     };
   }

packages/trigger-sdk/src/io.ts

@@ -199,6 +199,42 @@ export class IO {
     )) as TResponseData;
   }
 
+  /** `io.backgroundTask()` invokes a background task.
+   * @param key Should be a stable and unique key inside the `run()`. See [resumability](https://trigger.dev/docs/documentation/concepts/resumability) for more information.
+   * @param taskId The id of the background task to invoke.
+   * @param payload The payload to send to the background task.
+   */
+  async backgroundTask<TResultData>(
+    key: string | any[],
+    id: string,
+    version: string,
+    payload: any
+  ): Promise<TResultData> {
+    return (await this.runTask(
+      key,
+      async (task) => {
+        return task.output;
+      },
+      {
+        name: `invoke ${id}`,
+        params: { payload, id, version },
+        operation: "backgroundTask",
+        icon: "background",
+        noop: false,
+        properties: [
+          {
+            label: "Task ID",
+            text: id,
+          },
+          {
+            label: "Version",
+            text: version,
+          },
+        ],
+      }
+    )) as TResultData;
+  }
+
   /** `io.sendEvent()` allows you to send an event from inside a Job run. The sent even will trigger any Jobs that are listening for that event (based on the name).
    * @param key Should be a stable and unique key inside the `run()`. See [resumability](https://trigger.dev/docs/documentation/concepts/resumability) for more information.
    * @param event The event to send. The event name must match the name of the event that your Jobs are listening for.

packages/trigger-sdk/src/runLocalStorage.ts

@@ -0,0 +1,10 @@
+import { IO } from "./io";
+import { TriggerContext } from "./types";
+import { TypedAsyncLocalStorage } from "./utils/typedAsyncLocalStorage";
+
+export type RunStore = {
+  io: IO;
+  ctx: TriggerContext;
+};
+
+export const runLocalStorage = new TypedAsyncLocalStorage<RunStore>();

packages/trigger-sdk/src/triggerClient.ts

@@ -43,6 +43,7 @@ import type {
   TriggerPreprocessContext,
 } from "./types";
 import { BackgroundTask, BackgroundTaskOptions } from "./backgroundTask";
+import { runLocalStorage } from "./runLocalStorage";
 
 const registerSourceEvent: EventSpecification<RegisterSourceEventV2> = {
   name: REGISTER_SOURCE_EVENT_V2,
@@ -657,11 +658,13 @@ export class TriggerClient {
     );
 
     try {
-      const output = await job.options.run(
-        job.trigger.event.parsePayload(body.event.payload ?? {}),
-        ioWithConnections,
-        context
-      );
+      const output = await runLocalStorage.runWith({ io, ctx: context }, () => {
+        return job.options.run(
+          job.trigger.event.parsePayload(body.event.payload ?? {}),
+          ioWithConnections,
+          context
+        );
+      });
 
       return { status: "SUCCESS", output };
     } catch (error) {
@@ -873,8 +876,10 @@ export class TriggerClient {
     return new Job<TTrigger, TIntegrations>(this, options);
   }
 
-  defineBackgroundTask<TPayload = any>(options: BackgroundTaskOptions<TPayload>) {
-    return new BackgroundTask<TPayload>(this, options);
+  defineBackgroundTask<TPayload = any, TRunResult = any>(
+    options: BackgroundTaskOptions<TPayload, TRunResult>
+  ) {
+    return new BackgroundTask<TPayload, TRunResult>(this, options);
   }
 }
 

packages/trigger-sdk/src/utils/typedAsyncLocalStorage.ts

@@ -0,0 +1,17 @@
+import { AsyncLocalStorage } from "async_hooks";
+
+export class TypedAsyncLocalStorage<T> {
+  private storage: AsyncLocalStorage<T>;
+
+  constructor() {
+    this.storage = new AsyncLocalStorage<T>();
+  }
+
+  runWith<R extends (...args: any[]) => Promise<any>>(context: T, fn: R): Promise<ReturnType<R>> {
+    return this.storage.run(context, fn);
+  }
+
+  getStore(): T | undefined {
+    return this.storage.getStore();
+  }
+}

pnpm-lock.yaml

@@ -109,6 +109,8 @@ importers:
       '@trigger.dev/database': workspace:*
       '@trigger.dev/sdk': workspace:*
       '@trigger.dev/tailwind-config': workspace:*
+      '@types/archiver': ^5.3.2
+      '@types/async-retry': ^1.4.5
       '@types/bcryptjs': ^2.4.2
       '@types/compression': ^1.7.2
       '@types/eslint': ^8.4.6
@@ -124,6 +126,7 @@ importers:
       '@types/qs': ^6.9.7
       '@types/react': 18.2.17
       '@types/react-dom': 18.2.7
+      '@types/retry': ^0.12.2
       '@types/semver': ^7.3.13
       '@types/simple-oauth2': ^5.0.4
       '@types/slug': ^5.0.3
@@ -131,6 +134,8 @@ importers:
       '@typescript-eslint/eslint-plugin': ^5.59.6
       '@typescript-eslint/parser': ^5.59.6
       '@uiw/react-codemirror': ^4.19.5
+      archiver: ^6.0.1
+      async-retry: ^1.3.3
       autoprefixer: ^10.4.13
       class-variance-authority: ^0.5.2
       clsx: ^1.2.1
@@ -233,6 +238,8 @@ importers:
       '@trigger.dev/database': link:../../packages/database
       '@trigger.dev/sdk': link:../../packages/trigger-sdk
       '@uiw/react-codemirror': 4.19.5_k4ec5g7vuuzzonc3d6xbjnmmle
+      archiver: 6.0.1
+      async-retry: 1.3.3
       class-variance-authority: 0.5.2_typescript@4.9.4
       clsx: 1.2.1
       compression: 1.7.4
@@ -301,6 +308,8 @@ importers:
       '@tailwindcss/typography': 0.5.9_tailwindcss@3.3.2
       '@total-typescript/ts-reset': 0.4.2
       '@trigger.dev/tailwind-config': link:../../config-packages/tailwind-config
+      '@types/archiver': 5.3.2
+      '@types/async-retry': 1.4.5
       '@types/bcryptjs': 2.4.2
       '@types/compression': 1.7.2
       '@types/eslint': 8.4.10
@@ -316,6 +325,7 @@ importers:
       '@types/qs': 6.9.7
       '@types/react': 18.2.17
       '@types/react-dom': 18.2.7
+      '@types/retry': 0.12.2
       '@types/semver': 7.3.13
       '@types/simple-oauth2': 5.0.4
       '@types/slug': 5.0.3
@@ -879,6 +889,7 @@ importers:
       '@trigger.dev/core': workspace:*
       '@trigger.dev/tsconfig': workspace:*
       '@types/degit': ^2.8.3
+      '@types/dockerode': ^3.3.19
       '@types/gradient-string': ^1.1.2
       '@types/inquirer': ^9.0.3
       '@types/jest': ^29.5.3
@@ -886,10 +897,12 @@ importers:
       '@types/node': '16'
       '@types/node-fetch': ^2.6.2
       '@types/semver': ^7.3.13
+      '@types/tar': ^6.1.4
       chalk: ^5.2.0
       chokidar: ^3.5.3
       commander: ^9.4.1
       degit: ^2.8.4
+      dockerode: ^3.3.5
       dotenv: ^16.3.1
       esbuild: ^0.19.2
       execa: ^7.0.0
@@ -910,6 +923,7 @@ importers:
       semver: ^7.5.0
       simple-git: ^3.19.0
       spawn-please: ^2.0.2
+      tar: ^6.2.0
       terminal-link: ^3.0.0
       ts-jest: ^29.1.1
       tsconfck: ^2.1.2
@@ -926,6 +940,7 @@ importers:
       chokidar: 3.5.3
       commander: 9.5.0
       degit: 2.8.4
+      dockerode: 3.3.5
       dotenv: 16.3.1
       esbuild: 0.19.2
       execa: 7.0.0
@@ -944,6 +959,7 @@ importers:
       semver: 7.5.4
       simple-git: 3.19.0
       spawn-please: 2.0.2
+      tar: 6.2.0
       terminal-link: 3.0.0
       tsconfck: 2.1.2_typescript@4.9.5
       url: 0.11.1
@@ -951,6 +967,7 @@ importers:
     devDependencies:
       '@gmrchk/cli-testing-library': 0.1.2
       '@trigger.dev/tsconfig': link:../../config-packages/tsconfig
+      '@types/dockerode': 3.3.19
       '@types/gradient-string': 1.1.2
       '@types/inquirer': 9.0.3
       '@types/jest': 29.5.3
@@ -958,6 +975,7 @@ importers:
       '@types/node': 16.18.11
       '@types/node-fetch': 2.6.2
       '@types/semver': 7.5.1
+      '@types/tar': 6.1.4
       jest: 29.6.2_@types+node@16.18.11
       rimraf: 3.0.2
       ts-jest: 29.1.1_fqmqq3oqqfykqggnxm3kt3cose
@@ -3759,6 +3777,10 @@ packages:
       to-fast-properties: 2.0.0
     dev: true
 
+  /@balena/dockerignore/1.0.2:
+    resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==}
+    dev: false
+
   /@base2/pretty-print-object/1.0.1:
     resolution: {integrity: sha512-4iri8i1AqYHJE2DstZYkyEprg6Pq6sKx3xn5FpySk9sNhH7qN2LLlHJCfDTZRILNwQNPD7mATWM0TBui7uC1pA==}
     dev: true
@@ -5488,7 +5510,7 @@ packages:
     engines: {node: ^8.13.0 || >=10.10.0}
     dependencies:
       '@grpc/proto-loader': 0.7.7
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
     dev: false
 
   /@grpc/proto-loader/0.7.7:
@@ -5924,7 +5946,7 @@ packages:
     dependencies:
       '@types/istanbul-lib-coverage': 2.0.4
       '@types/istanbul-reports': 3.0.1
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       '@types/yargs': 16.0.5
       chalk: 4.1.2
     dev: true
@@ -5936,7 +5958,7 @@ packages:
       '@jest/schemas': 29.6.0
       '@types/istanbul-lib-coverage': 2.0.4
       '@types/istanbul-reports': 3.0.1
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       '@types/yargs': 17.0.19
       chalk: 4.1.2
     dev: true
@@ -11259,10 +11281,22 @@ packages:
       '@types/estree': 1.0.0
     dev: true
 
+  /@types/archiver/5.3.2:
+    resolution: {integrity: sha512-IctHreBuWE5dvBDz/0WeKtyVKVRs4h75IblxOACL92wU66v+HGAfEYAOyXkOFphvRJMhuXdI9huDXpX0FC6lCw==}
+    dependencies:
+      '@types/readdir-glob': 1.1.1
+    dev: true
+
   /@types/aria-query/5.0.1:
     resolution: {integrity: sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==}
     dev: true
 
+  /@types/async-retry/1.4.5:
+    resolution: {integrity: sha512-YrdjSD+yQv7h6d5Ip+PMxh3H6ZxKyQk0Ts+PvaNRInxneG9PFVZjFg77ILAN+N6qYf7g4giSJ1l+ZjQ1zeegvA==}
+    dependencies:
+      '@types/retry': 0.12.2
+    dev: true
+
   /@types/aws-lambda/8.10.114:
     resolution: {integrity: sha512-M8WpEGfC9iQ6V2Ccq6nGIXoQgeVc6z0Ngk8yCOL5V/TYIxshvb0MWQYLFFTZDesL0zmsoBc4OBjG9DB/4rei6w==}
     dev: false
@@ -11317,7 +11351,7 @@ packages:
   /@types/bunyan/1.8.7:
     resolution: {integrity: sha512-jaNt6xX5poSmXuDAkQrSqx2zkR66OrdRDuVnU8ldvn3k/Ci/7Sf5nooKspQWimDnw337Bzt/yirqSThTjvrHkg==}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
     dev: false
 
   /@types/cacheable-request/6.0.3:
@@ -11347,7 +11381,7 @@ packages:
   /@types/connect/3.4.35:
     resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
 
   /@types/content-disposition/0.5.5:
     resolution: {integrity: sha512-v6LCdKfK6BwcqMo+wYW05rLS12S0ZO0Fl4w1h4aaZMD7bqT3gVUns6FvLJKGZHQmYn3SX55JWGpziwJRwVgutA==}
@@ -11378,6 +11412,20 @@ packages:
     resolution: {integrity: sha512-xxgAGA2SAU4111QefXPSp5eGbDm/hW6zhvYl9IeEPZEry9F4d66QAHm5qpUXjb6IsevZV/7emAEx5MhP6O192g==}
     dev: true
 
+  /@types/docker-modem/3.0.3:
+    resolution: {integrity: sha512-i1A2Etnav7uHizZ87vUf4EqwJehY3JOcTfBS0pGBlO+HQ0jg2lUMCaJRg9VQM8ldZkpYdIfsenxcTOCpwxPXEg==}
+    dependencies:
+      '@types/node': 20.5.9
+      '@types/ssh2': 1.11.13
+    dev: true
+
+  /@types/dockerode/3.3.19:
+    resolution: {integrity: sha512-7CC5yIpQi+bHXwDK43b/deYXteP3Lem9gdocVVHJPSRJJLMfbiOchQV3rDmAPkMw+n3GIVj7m1six3JW+VcwwA==}
+    dependencies:
+      '@types/docker-modem': 3.0.3
+      '@types/node': 20.5.9
+    dev: true
+
   /@types/doctrine/0.0.3:
     resolution: {integrity: sha512-w5jZ0ee+HaPOaX25X2/2oGR/7rgAQSYII7X7pp0m9KgBfMP7uKfMfTvcpl5Dj+eDBbpxKGiqE+flqDr6XTd2RA==}
     dev: true
@@ -11463,13 +11511,13 @@ packages:
     resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==}
     dependencies:
       '@types/minimatch': 5.1.2
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
 
   /@types/glob/8.1.0:
     resolution: {integrity: sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==}
     dependencies:
       '@types/minimatch': 5.1.2
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
     dev: true
 
   /@types/graceful-fs/4.1.6:
@@ -11497,7 +11545,7 @@ packages:
       '@types/hapi__catbox': 10.2.4
       '@types/hapi__mimos': 4.1.4
       '@types/hapi__shot': 4.1.2
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       joi: 17.7.0
     dev: false
 
@@ -11642,7 +11690,7 @@ packages:
       '@types/http-errors': 2.0.1
       '@types/keygrip': 1.0.2
       '@types/koa-compose': 3.2.5
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
     dev: false
 
   /@types/koa__router/8.0.7:
@@ -11690,7 +11738,7 @@ packages:
   /@types/memcached/2.2.7:
     resolution: {integrity: sha512-ImJbz1i8pl+OnyhYdIDnHe8jAuM8TOwM/7VsciqhYX3IL0jPPUToAtVxklfcWFGYckahEYZxhd9FS0z3MM1dpA==}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
     dev: false
 
   /@types/mime-db/1.43.1:
@@ -11723,7 +11771,7 @@ packages:
   /@types/mysql/2.15.19:
     resolution: {integrity: sha512-wSRg2QZv14CWcZXkgdvHbbV2ACufNy5EgI8mBBxnJIptchv7DBy/h53VMa2jDhyo0C9MO4iowE6z9vF8Ja1DkQ==}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
     dev: false
 
   /@types/node-fetch/2.6.2:
@@ -11736,7 +11784,7 @@ packages:
   /@types/node-fetch/2.6.4:
     resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       form-data: 3.0.1
 
   /@types/node/12.20.55:
@@ -11798,7 +11846,7 @@ packages:
   /@types/pg/8.6.1:
     resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       pg-protocol: 1.6.0
       pg-types: 2.2.0
     dev: false
@@ -11844,6 +11892,12 @@ packages:
       '@types/scheduler': 0.16.2
       csstype: 3.1.1
 
+  /@types/readdir-glob/1.1.1:
+    resolution: {integrity: sha512-ImM6TmoF8bgOwvehGviEj3tRdRBbQujr1N+0ypaln/GWjaerOB26jb93vsRHmdMtvVQZQebOlqt2HROark87mQ==}
+    dependencies:
+      '@types/node': 20.5.9
+    dev: true
+
   /@types/responselike/1.0.0:
     resolution: {integrity: sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==}
     dependencies:
@@ -11853,6 +11907,10 @@ packages:
     resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==}
     dev: false
 
+  /@types/retry/0.12.2:
+    resolution: {integrity: sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==}
+    dev: true
+
   /@types/scheduler/0.16.2:
     resolution: {integrity: sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==}
 
@@ -11891,6 +11949,12 @@ packages:
     resolution: {integrity: sha512-yPX0bb1SvrpaGlHuSiz6EicgRI4VBE+LO7IANlZagQwtaoKjLLcZc8y6s13vKp41mYvMCSzjtObxvU7/0JRPaA==}
     dev: true
 
+  /@types/ssh2/1.11.13:
+    resolution: {integrity: sha512-08WbG68HvQ2YVi74n2iSUnYHYpUdFc/s2IsI0BHBdJwaqYJpWlVv9elL0tYShTv60yr0ObdxJR5NrCRiGJ/0CQ==}
+    dependencies:
+      '@types/node': 18.17.1
+    dev: true
+
   /@types/stack-utils/2.0.1:
     resolution: {integrity: sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==}
     dev: true
@@ -11898,14 +11962,14 @@ packages:
   /@types/tar/6.1.4:
     resolution: {integrity: sha512-Cp4oxpfIzWt7mr2pbhHT2OTXGMAL0szYCzuf8lRWyIMCgsx6/Hfc3ubztuhvzXHXgraTQxyOCmmg7TDGIMIJJQ==}
     dependencies:
-      '@types/node': 18.15.13
+      '@types/node': 20.5.9
       minipass: 4.0.0
     dev: true
 
   /@types/tedious/4.0.9:
     resolution: {integrity: sha512-ipwFvfy9b2m0gjHsIX0D6NAAwGCKokzf5zJqUZHUGt+7uWVlBIy6n2eyMgiKQ8ChLFVxic/zwQUhjLYNzbHDRA==}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
     dev: false
 
   /@types/through/0.0.30:
@@ -12816,6 +12880,31 @@ packages:
   /aproba/2.0.0:
     resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==}
 
+  /archiver-utils/4.0.1:
+    resolution: {integrity: sha512-Q4Q99idbvzmgCTEAAhi32BkOyq8iVI5EwdO0PmBDSGIzzjYNdcFn7Q7k3OzbLy4kLUPXfJtG6fO2RjftXbobBg==}
+    engines: {node: '>= 12.0.0'}
+    dependencies:
+      glob: 8.1.0
+      graceful-fs: 4.2.10
+      lazystream: 1.0.1
+      lodash: 4.17.21
+      normalize-path: 3.0.0
+      readable-stream: 3.6.0
+    dev: false
+
+  /archiver/6.0.1:
+    resolution: {integrity: sha512-CXGy4poOLBKptiZH//VlWdFuUC1RESbdZjGjILwBuZ73P7WkAUN0htfSfBq/7k6FRFlpu7bg4JOkj1vU9G6jcQ==}
+    engines: {node: '>= 12.0.0'}
+    dependencies:
+      archiver-utils: 4.0.1
+      async: 3.2.4
+      buffer-crc32: 0.2.13
+      readable-stream: 3.6.0
+      readdir-glob: 1.1.3
+      tar-stream: 3.1.6
+      zip-stream: 5.0.1
+    dev: false
+
   /are-we-there-yet/2.0.0:
     resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==}
     engines: {node: '>=10'}
@@ -12974,6 +13063,12 @@ packages:
     engines: {node: '>=8'}
     dev: false
 
+  /asn1/0.2.6:
+    resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==}
+    dependencies:
+      safer-buffer: 2.1.2
+    dev: false
+
   /assert/2.0.0:
     resolution: {integrity: sha512-se5Cd+js9dXJnu6Ag2JFc00t+HmHOen+8Q+L7O9zI0PqQXr20uk2J0XQqMxZEeo5U50o8Nvmmx7dZrl+Ufr35A==}
     dependencies:
@@ -13032,9 +13127,14 @@ packages:
     resolution: {integrity: sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==}
     dev: true
 
+  /async-retry/1.3.3:
+    resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
+    dependencies:
+      retry: 0.13.1
+    dev: false
+
   /async/3.2.4:
     resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==}
-    dev: true
 
   /asynckit/0.4.0:
     resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
@@ -13128,7 +13228,7 @@ packages:
   /axios/0.21.4_debug@4.3.2:
     resolution: {integrity: sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==}
     dependencies:
-      follow-redirects: 1.15.2_debug@4.3.2
+      follow-redirects: 1.15.2
     transitivePeerDependencies:
       - debug
     dev: false
@@ -13164,6 +13264,10 @@ packages:
     dependencies:
       deep-equal: 2.2.0
 
+  /b4a/1.6.4:
+    resolution: {integrity: sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==}
+    dev: false
+
   /babel-core/7.0.0-bridge.0_@babel+core@7.21.8:
     resolution: {integrity: sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==}
     peerDependencies:
@@ -13385,6 +13489,12 @@ packages:
     engines: {node: '>=10.0.0'}
     dev: true
 
+  /bcrypt-pbkdf/1.0.2:
+    resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==}
+    dependencies:
+      tweetnacl: 0.14.5
+    dev: false
+
   /before-after-hook/2.2.3:
     resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==}
     dev: false
@@ -13687,6 +13797,12 @@ packages:
       xtend: 4.0.2
     dev: false
 
+  /buildcheck/0.0.6:
+    resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==}
+    engines: {node: '>=10.0.0'}
+    dev: false
+    optional: true
+
   /builtins/1.0.3:
     resolution: {integrity: sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==}
     dev: true
@@ -13805,7 +13921,7 @@ packages:
       minipass-pipeline: 1.2.4
       p-map: 4.0.0
       ssri: 10.0.5
-      tar: 6.1.13
+      tar: 6.2.0
       unique-filename: 3.0.0
     dev: false
 
@@ -14048,7 +14164,6 @@ packages:
 
   /chownr/1.1.4:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
-    dev: true
 
   /chownr/2.0.0:
     resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==}
@@ -14313,6 +14428,16 @@ packages:
     resolution: {integrity: sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==}
     dev: false
 
+  /compress-commons/5.0.1:
+    resolution: {integrity: sha512-MPh//1cERdLtqwO3pOFLeXtpuai0Y2WCd5AhtKxznqM7WtaMYaOEMSgn45d9D10sIHSfIKE603HlOp8OPGrvag==}
+    engines: {node: '>= 12.0.0'}
+    dependencies:
+      crc-32: 1.2.2
+      crc32-stream: 5.0.0
+      normalize-path: 3.0.0
+      readable-stream: 3.6.0
+    dev: false
+
   /compressible/2.0.18:
     resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==}
     engines: {node: '>= 0.6'}
@@ -14480,6 +14605,16 @@ packages:
       p-event: 4.2.0
     dev: false
 
+  /cpu-features/0.0.9:
+    resolution: {integrity: sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==}
+    engines: {node: '>=10.0.0'}
+    requiresBuild: true
+    dependencies:
+      buildcheck: 0.0.6
+      nan: 2.17.0
+    dev: false
+    optional: true
+
   /cpy/8.1.2:
     resolution: {integrity: sha512-dmC4mUesv0OYH2kNFEidtf/skUwv4zePmGeepjyyJ0qTo5+8KhA1o99oIAwVVLzQMAeDJml74d6wPPKb6EZUTg==}
     engines: {node: '>=8'}
@@ -14497,6 +14632,20 @@ packages:
       - supports-color
     dev: false
 
+  /crc-32/1.2.2:
+    resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==}
+    engines: {node: '>=0.8'}
+    hasBin: true
+    dev: false
+
+  /crc32-stream/5.0.0:
+    resolution: {integrity: sha512-B0EPa1UK+qnpBZpG+7FgPCu0J2ETLpXq09o9BkLkEAhdB6Z61Qo4pJ3JYu0c+Qi+/SAL7QThqnzS06pmSSyZaw==}
+    engines: {node: '>= 12.0.0'}
+    dependencies:
+      crc-32: 1.2.2
+      readable-stream: 3.6.0
+    dev: false
+
   /create-require/1.1.1:
     resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
 
@@ -15092,6 +15241,29 @@ packages:
   /dlv/1.1.3:
     resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==}
 
+  /docker-modem/3.0.8:
+    resolution: {integrity: sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ==}
+    engines: {node: '>= 8.0'}
+    dependencies:
+      debug: 4.3.4
+      readable-stream: 3.6.0
+      split-ca: 1.0.1
+      ssh2: 1.14.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  /dockerode/3.3.5:
+    resolution: {integrity: sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA==}
+    engines: {node: '>= 8.0'}
+    dependencies:
+      '@balena/dockerignore': 1.0.2
+      docker-modem: 3.0.8
+      tar-fs: 2.0.1
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /doctrine/2.1.0:
     resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
     engines: {node: '>=0.10.0'}
@@ -17447,6 +17619,10 @@ packages:
   /fast-deep-equal/3.1.3:
     resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
 
+  /fast-fifo/1.3.2:
+    resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==}
+    dev: false
+
   /fast-folder-size/1.6.1:
     resolution: {integrity: sha512-F3tRpfkAzb7TT2JNKaJUglyuRjRa+jelQD94s9OSqkfEeytLmupCqQiD+H2KoIXGtp4pB5m4zNmv5m2Ktcr+LA==}
     hasBin: true
@@ -17703,18 +17879,6 @@ packages:
       debug:
         optional: true
 
-  /follow-redirects/1.15.2_debug@4.3.2:
-    resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
-    engines: {node: '>=4.0'}
-    peerDependencies:
-      debug: '*'
-    peerDependenciesMeta:
-      debug:
-        optional: true
-    dependencies:
-      debug: 4.3.2
-    dev: false
-
   /for-each/0.3.3:
     resolution: {integrity: sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==}
     dependencies:
@@ -17881,7 +18045,6 @@ packages:
 
   /fs-constants/1.0.0:
     resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
-    dev: true
 
   /fs-extra/10.1.0:
     resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==}
@@ -19750,7 +19913,7 @@ packages:
     dependencies:
       '@jest/types': 29.6.1
       '@types/graceful-fs': 4.1.6
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       anymatch: 3.1.3
       fb-watchman: 2.0.2
       graceful-fs: 4.2.10
@@ -19966,7 +20129,7 @@ packages:
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
     dependencies:
       '@jest/types': 29.6.1
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       chalk: 4.1.2
       ci-info: 3.7.1
       graceful-fs: 4.2.10
@@ -20003,7 +20166,7 @@ packages:
     resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==}
     engines: {node: '>= 10.13.0'}
     dependencies:
-      '@types/node': 20.5.2
+      '@types/node': 20.5.9
       merge-stream: 2.0.0
       supports-color: 8.1.1
     dev: true
@@ -20359,6 +20522,13 @@ packages:
       dotenv-expand: 10.0.0
     dev: true
 
+  /lazystream/1.0.1:
+    resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==}
+    engines: {node: '>= 0.6.3'}
+    dependencies:
+      readable-stream: 2.3.7
+    dev: false
+
   /leac/0.6.0:
     resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==}
     dev: false
@@ -21451,6 +21621,7 @@ packages:
     engines: {node: '>=8'}
     dependencies:
       yallist: 4.0.0
+    dev: true
 
   /minipass/5.0.0:
     resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==}
@@ -21484,7 +21655,6 @@ packages:
 
   /mkdirp-classic/0.5.3:
     resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
-    dev: true
 
   /mkdirp/0.5.6:
     resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==}
@@ -21619,6 +21789,11 @@ packages:
       object-assign: 4.1.1
       thenify-all: 1.6.0
 
+  /nan/2.17.0:
+    resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==}
+    dev: false
+    optional: true
+
   /nano-css/5.3.5_biqbaboplfbrettd7655fr4n2y:
     resolution: {integrity: sha512-vSB9X12bbNu4ALBu7nigJgRViZ6ja3OU7CeuiV1zMIbXOdmkLahgtPmh3GBOlDxbKY0CitqlPdOReGlBLSp+yg==}
     peerDependencies:
@@ -22096,7 +22271,7 @@ packages:
       npmlog: 6.0.2
       rimraf: 3.0.2
       semver: 7.5.4
-      tar: 6.1.13
+      tar: 6.2.0
       which: 2.0.2
     transitivePeerDependencies:
       - supports-color
@@ -22842,7 +23017,7 @@ packages:
       read-package-json-fast: 3.0.2
       sigstore: 1.9.0
       ssri: 10.0.5
-      tar: 6.1.13
+      tar: 6.2.0
     transitivePeerDependencies:
       - bluebird
       - supports-color
@@ -23885,6 +24060,10 @@ packages:
   /queue-microtask/1.2.3:
     resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==}
 
+  /queue-tick/1.0.1:
+    resolution: {integrity: sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==}
+    dev: false
+
   /quick-lru/4.0.1:
     resolution: {integrity: sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==}
     engines: {node: '>=8'}
@@ -24282,6 +24461,12 @@ packages:
       string_decoder: 1.3.0
       util-deprecate: 1.0.2
 
+  /readdir-glob/1.1.3:
+    resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==}
+    dependencies:
+      minimatch: 5.1.2
+    dev: false
+
   /readdirp/3.6.0:
     resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
     engines: {node: '>=8.10.0'}
@@ -25439,6 +25624,10 @@ packages:
   /spdx-license-ids/3.0.12:
     resolution: {integrity: sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA==}
 
+  /split-ca/1.0.1:
+    resolution: {integrity: sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==}
+    dev: false
+
   /split-string/3.1.0:
     resolution: {integrity: sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==}
     engines: {node: '>=0.10.0'}
@@ -25454,6 +25643,18 @@ packages:
   /sprintf-js/1.0.3:
     resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==}
 
+  /ssh2/1.14.0:
+    resolution: {integrity: sha512-AqzD1UCqit8tbOKoj6ztDDi1ffJZ2rV2SwlgrVVrHPkV5vWqGJOVp5pmtj18PunkPJAuKQsnInyKV+/Nb2bUnA==}
+    engines: {node: '>=10.16.0'}
+    requiresBuild: true
+    dependencies:
+      asn1: 0.2.6
+      bcrypt-pbkdf: 1.0.2
+    optionalDependencies:
+      cpu-features: 0.0.9
+      nan: 2.17.0
+    dev: false
+
   /ssri/10.0.5:
     resolution: {integrity: sha512-bSf16tAFkGeRlUNDjXu8FzaMQt6g2HZJrun7mtMbIPOddxt3GLMSz5VWUWcqTJUPfLEaDIepGxv+bYQW49596A==}
     engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -25600,6 +25801,13 @@ packages:
     resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==}
     engines: {node: '>=10.0.0'}
 
+  /streamx/2.15.1:
+    resolution: {integrity: sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==}
+    dependencies:
+      fast-fifo: 1.3.2
+      queue-tick: 1.0.1
+    dev: false
+
   /strict-event-emitter/0.2.8:
     resolution: {integrity: sha512-KDf/ujU8Zud3YaLtMCcTI4xkZlZVIYxTLr+XIULexP+77EEVWixeXroLUXQXiVtH4XH2W7jr/3PT1v3zBuvc3A==}
     dependencies:
@@ -26041,6 +26249,15 @@ packages:
     resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==}
     engines: {node: '>=6'}
 
+  /tar-fs/2.0.1:
+    resolution: {integrity: sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==}
+    dependencies:
+      chownr: 1.1.4
+      mkdirp-classic: 0.5.3
+      pump: 3.0.0
+      tar-stream: 2.2.0
+    dev: false
+
   /tar-fs/2.1.1:
     resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==}
     dependencies:
@@ -26059,7 +26276,14 @@ packages:
       fs-constants: 1.0.0
       inherits: 2.0.4
       readable-stream: 3.6.0
-    dev: true
+
+  /tar-stream/3.1.6:
+    resolution: {integrity: sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==}
+    dependencies:
+      b4a: 1.6.4
+      fast-fifo: 1.3.2
+      streamx: 2.15.1
+    dev: false
 
   /tar/6.1.13:
     resolution: {integrity: sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==}
@@ -26071,6 +26295,19 @@ packages:
       minizlib: 2.1.2
       mkdirp: 1.0.4
       yallist: 4.0.0
+    dev: true
+
+  /tar/6.2.0:
+    resolution: {integrity: sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==}
+    engines: {node: '>=10'}
+    dependencies:
+      chownr: 2.0.0
+      fs-minipass: 2.1.0
+      minipass: 5.0.0
+      minizlib: 2.1.2
+      mkdirp: 1.0.4
+      yallist: 4.0.0
+    dev: false
 
   /telejson/7.1.0:
     resolution: {integrity: sha512-jFJO4P5gPebZAERPkJsqMAQ0IMA1Hi0AoSfxpnUaV6j6R2SZqlpkbS20U6dEUtA3RUYt2Ak/mTlkQzHH9Rv/hA==}
@@ -26932,6 +27169,10 @@ packages:
       turbo-windows-arm64: 1.10.3
     dev: true
 
+  /tweetnacl/0.14.5:
+    resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==}
+    dev: false
+
   /type-check/0.3.2:
     resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==}
     engines: {node: '>= 0.8.0'}
@@ -28351,6 +28592,15 @@ packages:
     engines: {node: '>=12.20'}
     dev: true
 
+  /zip-stream/5.0.1:
+    resolution: {integrity: sha512-UfZ0oa0C8LI58wJ+moL46BDIMgCQbnsb+2PoiJYtonhBsMh2bq1eRBVkvjfVsqbEHd9/EgKPUuL9saSSsec8OA==}
+    engines: {node: '>= 12.0.0'}
+    dependencies:
+      archiver-utils: 4.0.1
+      compress-commons: 5.0.1
+      readable-stream: 3.6.0
+    dev: false
+
   /zod-error/1.5.0:
     resolution: {integrity: sha512-zzopKZ/skI9iXpqCEPj+iLCKl9b88E43ehcU+sbRoHuwGd9F1IDVGQ70TyO6kmfiRL1g4IXkjsXK+g1gLYl4WQ==}
     dependencies: