WIP (fly.io multi-tenant idea)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/config.json

@@ -1,11 +1,27 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@2.2.0/schema.json",
-  "changelog": "@changesets/cli/changelog",
+  "changelog": [
+    "@remix-run/changelog-github",
+    {
+      "repo": "triggerdotdev/trigger.dev"
+    }
+  ],
   "commit": false,
-  "fixed": [["@trigger.dev/*"]],
+  "fixed": [
+    [
+      "@trigger.dev/*"
+    ]
+  ],
   "linked": [],
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
-  "ignore": ["webapp", "emails", "@trigger.dev/database"]
-}
+  "ignore": [
+    "webapp",
+    "emails",
+    "@trigger.dev/database"
+  ],
+  "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
+    "onlyUpdatePeerDependentsWhenOutOfRange": true
+  }
+}

.changeset/witty-weeks-fly.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Background tasks

.github/workflows/publish.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - improvements/*
     tags:
       - "v.docker.*"
     paths:
@@ -95,6 +96,12 @@ jobs:
     name: e2e Tests
     runs-on: buildjet-4vcpu-ubuntu-2204
     steps:
+      - name: 🐳 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v3
         with:
@@ -154,6 +161,11 @@ jobs:
       version: ${{ steps.get_version.outputs.version }}
       short_sha: ${{ steps.get_commit.outputs.sha_short }}
     steps:
+      - name: 🐳 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v3
 
@@ -167,6 +179,10 @@ jobs:
               IMAGE_TAG="v${ORIGINAL_VERSION}"
             fi
             echo "IMAGE_TAG=${IMAGE_TAG}"
+          elif [[ $GITHUB_REF == refs/heads/improvements/* ]]; then
+            ORIGINAL_VERSION="${GITHUB_REF#refs/heads/improvements/}"
+            IMAGE_TAG="${ORIGINAL_VERSION}.rc"
+            echo "IMAGE_TAG=${IMAGE_TAG}"
           elif [[ $GITHUB_REF == refs/heads/* ]]; then
             IMAGE_TAG="${GITHUB_REF#refs/heads/}"
             echo "IMAGE_TAG=${IMAGE_TAG}"

.github/workflows/release.yml

@@ -8,7 +8,7 @@ on:
       - ".github/workflows/release.yml"
       - "packages/**"
       - "!packages/**/*.md"
-      - "changesets/**"
+      - ".changeset/**"
       - "integrations/**"
       - "!integrations/**/*.md"
       - "pnpm-lock.yaml"

.vscode/launch.json

@@ -5,29 +5,28 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "command": "pnpm run dev --filter webapp",
-      "name": "Run webapp",
-      "request": "launch",
       "type": "node-terminal",
-      "cwd": "${workspaceFolder}"
+      "request": "launch",
+      "name": "Debug WebApp",
+      "command": "pnpm run dev --filter webapp",
+      "envFile": "${workspaceFolder}/apps/webapp/.env",
+      "cwd": "${workspaceFolder}",
+      "sourceMaps": true
     },
     {
       "type": "chrome",
       "request": "launch",
       "name": "Chrome webapp",
-      "url": "http://localhost:3000",
+      "url": "http://localhost:3030",
       "webRoot": "${workspaceFolder}/apps/webapp/app"
     },
     {
-      "type": "node",
+      "type": "node-terminal",
       "request": "launch",
-      "name": "Debug Current Test File",
-      "autoAttachChildProcesses": true,
-      "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
-      "program": "${workspaceRoot}/node_modules/vitest/vitest.mjs",
-      "args": ["run", "${relativeFile}"],
-      "smartStep": true,
-      "console": "integratedTerminal"
+      "name": "Debug Deploy CLI",
+      "command": "pnpm exec trigger-cli deploy --tag 0.0.0-background-tasks-20230906212613",
+      "cwd": "${workspaceFolder}/examples/nextjs-background-tasks",
+      "sourceMaps": true
     }
   ]
 }

CONTRIBUTING.md

@@ -136,38 +136,40 @@ pnpm run dev
 3. Create a new temporary Next.js app in examples directory
 
 ```sh
-pnpm create next-app@latest
+cd ./examples
+pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
 ```
 
-Follow the prompts to create a TypeScript project using the App Directory.
-
 4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
 
 ```json
 {
+  // other package.json properties
   "devDependencies": { "@trigger.dev/cli": "workspace:*" }
 }
 ```
 
-5. Open a new terminal window, navigate into the example, and initialize the CLI:
+5. Back in the terminal, navigate into the example, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
 
 ```sh
-cd examples/your-newly-created-nextjs-project
+cd ./test-cli
 pnpm i
 pnpm exec trigger-cli init
 ```
 
-6. When prompted, select `self-hosted` and enter `localhost:3030` for your local version of the webapp. When asked for an API key, use the key you copied earlier.
+6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
 
-7. Run the CLI
+```sh
+pnpm run dev
+```
+
+7. Open a new terminal window, and then run the `dev` command like so:
 
 ```sh
 pnpm exec trigger-cli dev
 ```
 
-8. After running the CLI, start your newly created Next.js project. You should now be able to see the changes.
-
-9. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
 
 ## Running end-to-end webapp tests
 

LICENSE

@@ -2,180 +2,180 @@
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-   1. Definitions.
+1.  Definitions.
 
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
+    "License" shall mean the terms and conditions for use, reproduction,
+    and distribution as defined by Sections 1 through 9 of this document.
 
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
+    "Licensor" shall mean the copyright owner or entity authorized by
+    the copyright owner that is granting the License.
 
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
+    "Legal Entity" shall mean the union of the acting entity and all
+    other entities that control, are controlled by, or are under common
+    control with that entity. For the purposes of this definition,
+    "control" means (i) the power, direct or indirect, to cause the
+    direction or management of such entity, whether by contract or
+    otherwise, or (ii) ownership of fifty percent (50%) or more of the
+    outstanding shares, or (iii) beneficial ownership of such entity.
 
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
+    "You" (or "Your") shall mean an individual or Legal Entity
+    exercising permissions granted by this License.
 
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
+    "Source" form shall mean the preferred form for making modifications,
+    including but not limited to software source code, documentation
+    source, and configuration files.
 
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
+    "Object" form shall mean any form resulting from mechanical
+    transformation or translation of a Source form, including but
+    not limited to compiled object code, generated documentation,
+    and conversions to other media types.
 
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
+    "Work" shall mean the work of authorship, whether in Source or
+    Object form, made available under the License, as indicated by a
+    copyright notice that is included in or attached to the work
+    (an example is provided in the Appendix below).
 
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
+    "Derivative Works" shall mean any work, whether in Source or Object
+    form, that is based on (or derived from) the Work and for which the
+    editorial revisions, annotations, elaborations, or other modifications
+    represent, as a whole, an original work of authorship. For the purposes
+    of this License, Derivative Works shall not include works that remain
+    separable from, or merely link (or bind by name) to the interfaces of,
+    the Work and Derivative Works thereof.
 
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
+    "Contribution" shall mean any work of authorship, including
+    the original version of the Work and any modifications or additions
+    to that Work or Derivative Works thereof, that is intentionally
+    submitted to Licensor for inclusion in the Work by the copyright owner
+    or by an individual or Legal Entity authorized to submit on behalf of
+    the copyright owner. For the purposes of this definition, "submitted"
+    means any form of electronic, verbal, or written communication sent
+    to the Licensor or its representatives, including but not limited to
+    communication on electronic mailing lists, source code control systems,
+    and issue tracking systems that are managed by, or on behalf of, the
+    Licensor for the purpose of discussing and improving the Work, but
+    excluding communication that is conspicuously marked or otherwise
+    designated in writing by the copyright owner as "Not a Contribution."
 
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
+    "Contributor" shall mean Licensor and any individual or Legal Entity
+    on behalf of whom a Contribution has been received by Licensor and
+    subsequently incorporated within the Work.
 
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
+2.  Grant of Copyright License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    copyright license to reproduce, prepare Derivative Works of,
+    publicly display, publicly perform, sublicense, and distribute the
+    Work and such Derivative Works in Source or Object form.
 
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
+3.  Grant of Patent License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    (except as stated in this section) patent license to make, have made,
+    use, offer to sell, sell, import, and otherwise transfer the Work,
+    where such license applies only to those patent claims licensable
+    by such Contributor that are necessarily infringed by their
+    Contribution(s) alone or by combination of their Contribution(s)
+    with the Work to which such Contribution(s) was submitted. If You
+    institute patent litigation against any entity (including a
+    cross-claim or counterclaim in a lawsuit) alleging that the Work
+    or a Contribution incorporated within the Work constitutes direct
+    or contributory patent infringement, then any patent licenses
+    granted to You under this License for that Work shall terminate
+    as of the date such litigation is filed.
 
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
+4.  Redistribution. You may reproduce and distribute copies of the
+    Work or Derivative Works thereof in any medium, with or without
+    modifications, and in Source or Object form, provided that You
+    meet the following conditions:
 
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
+    (a) You must give any other recipients of the Work or
+    Derivative Works a copy of this License; and
 
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
+    (b) You must cause any modified files to carry prominent notices
+    stating that You changed the files; and
 
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
+    (c) You must retain, in the Source form of any Derivative Works
+    that You distribute, all copyright, patent, trademark, and
+    attribution notices from the Source form of the Work,
+    excluding those notices that do not pertain to any part of
+    the Derivative Works; and
 
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
+    (d) If the Work includes a "NOTICE" text file as part of its
+    distribution, then any Derivative Works that You distribute must
+    include a readable copy of the attribution notices contained
+    within such NOTICE file, excluding those notices that do not
+    pertain to any part of the Derivative Works, in at least one
+    of the following places: within a NOTICE text file distributed
+    as part of the Derivative Works; within the Source form or
+    documentation, if provided along with the Derivative Works; or,
+    within a display generated by the Derivative Works, if and
+    wherever such third-party notices normally appear. The contents
+    of the NOTICE file are for informational purposes only and
+    do not modify the License. You may add Your own attribution
+    notices within Derivative Works that You distribute, alongside
+    or as an addendum to the NOTICE text from the Work, provided
+    that such additional attribution notices cannot be construed
+    as modifying the License.
 
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
+    You may add Your own copyright statement to Your modifications and
+    may provide additional or different license terms and conditions
+    for use, reproduction, or distribution of Your modifications, or
+    for any such Derivative Works as a whole, provided Your use,
+    reproduction, and distribution of the Work otherwise complies with
+    the conditions stated in this License.
 
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
+5.  Submission of Contributions. Unless You explicitly state otherwise,
+    any Contribution intentionally submitted for inclusion in the Work
+    by You to the Licensor shall be under the terms and conditions of
+    this License, without any additional terms or conditions.
+    Notwithstanding the above, nothing herein shall supersede or modify
+    the terms of any separate license agreement you may have executed
+    with Licensor regarding such Contributions.
 
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
+6.  Trademarks. This License does not grant permission to use the trade
+    names, trademarks, service marks, or product names of the Licensor,
+    except as required for reasonable and customary use in describing the
+    origin of the Work and reproducing the content of the NOTICE file.
 
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
+7.  Disclaimer of Warranty. Unless required by applicable law or
+    agreed to in writing, Licensor provides the Work (and each
+    Contributor provides its Contributions) on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+    implied, including, without limitation, any warranties or conditions
+    of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+    PARTICULAR PURPOSE. You are solely responsible for determining the
+    appropriateness of using or redistributing the Work and assume any
+    risks associated with Your exercise of permissions under this License.
 
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
+8.  Limitation of Liability. In no event and under no legal theory,
+    whether in tort (including negligence), contract, or otherwise,
+    unless required by applicable law (such as deliberate and grossly
+    negligent acts) or agreed to in writing, shall any Contributor be
+    liable to You for damages, including any direct, indirect, special,
+    incidental, or consequential damages of any character arising as a
+    result of this License or out of the use or inability to use the
+    Work (including but not limited to damages for loss of goodwill,
+    work stoppage, computer failure or malfunction, or any and all
+    other commercial damages or losses), even if such Contributor
+    has been advised of the possibility of such damages.
 
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
+9.  Accepting Warranty or Additional Liability. While redistributing
+    the Work or Derivative Works thereof, You may choose to offer,
+    and charge a fee for, acceptance of support, warranty, indemnity,
+    or other liability obligations and/or rights consistent with this
+    License. However, in accepting such obligations, You may act only
+    on Your own behalf and on Your sole responsibility, not on behalf
+    of any other Contributor, and only if You agree to indemnify,
+    defend, and hold each Contributor harmless for any liability
+    incurred by, or claims asserted against, such Contributor by reason
+    of your accepting any such warranty or additional liability.
 
-   END OF TERMS AND CONDITIONS
+END OF TERMS AND CONDITIONS
 
-   APPENDIX: How to apply the Apache License to your work.
+APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
       boilerplate notice, with the fields enclosed by brackets "[]"
@@ -186,16 +186,16 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+Copyright [2023] [Trigger.dev]
 
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
        http://www.apache.org/licenses/LICENSE-2.0
 
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -62,4 +62,4 @@ We provide an official trigger.dev docker image you can use to easily self-host
 
 ## Development
 
-To setup and develop locally or contribute to the open source project, follow our [developement guide](./CONTRIBUTING.md).
+To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).

apps/webapp/app/api.server.ts

@@ -0,0 +1,15 @@
+import { ApiEventLog } from "@trigger.dev/core";
+import { EventRecord } from "@trigger.dev/database";
+
+export function eventRecordToApiJson(eventRecord: EventRecord): ApiEventLog {
+  return {
+    id: eventRecord.eventId,
+    name: eventRecord.name,
+    payload: eventRecord.payload as any,
+    context: eventRecord.context as any,
+    timestamp: eventRecord.timestamp,
+    deliverAt: eventRecord.deliverAt,
+    deliveredAt: eventRecord.deliveredAt,
+    cancelledAt: eventRecord.cancelledAt,
+  };
+}

apps/webapp/app/components/ActiveBadge.tsx

@@ -0,0 +1,57 @@
+import { cn } from "~/utils/cn";
+
+const variant = {
+  small:
+    "py-[0.25rem] px-1.5 text-xxs font-normal inline-flex items-center justify-center whitespace-nowrap rounded-[0.125rem]",
+  normal:
+    "py-1 px-1.5 text-xs font-normal inline-flex items-center justify-center whitespace-nowrap rounded-sm",
+};
+
+type ActiveBadgeProps = {
+  active: boolean;
+  className?: string;
+  badgeSize?: keyof typeof variant;
+};
+
+export function ActiveBadge({ active, className, badgeSize = "normal" }: ActiveBadgeProps) {
+  switch (active) {
+    case true:
+      return (
+        <span className={cn(variant[badgeSize], "bg-slate-800 text-green-500", className)}>
+          Active
+        </span>
+      );
+    case false:
+      return (
+        <span className={cn(variant[badgeSize], "bg-slate-800 text-dimmed", className)}>
+          Disabled
+        </span>
+      );
+  }
+}
+
+export function MissingIntegrationBadge({
+  className,
+  badgeSize = "normal",
+}: {
+  className?: string;
+  badgeSize?: keyof typeof variant;
+}) {
+  return (
+    <span className={cn(variant[badgeSize], "bg-rose-600 text-white", className)}>
+      Missing Integration
+    </span>
+  );
+}
+
+export function NewBadge({
+  className,
+  badgeSize = "normal",
+}: {
+  className?: string;
+  badgeSize?: keyof typeof variant;
+}) {
+  return (
+    <span className={cn(variant[badgeSize], "bg-green-600 text-background", className)}>New!</span>
+  );
+}

apps/webapp/app/components/JobsStatusTable.tsx

@@ -0,0 +1,54 @@
+import { RuntimeEnvironmentType } from "@trigger.dev/database";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHeader,
+  TableHeaderCell,
+  TableRow,
+} from "~/components/primitives/Table";
+import { EnvironmentLabel } from "./environments/EnvironmentLabel";
+import { DateTime } from "./primitives/DateTime";
+import { ActiveBadge } from "./ActiveBadge";
+
+export type JobEnvironment = {
+  type: RuntimeEnvironmentType;
+  lastRun?: Date;
+  version: string;
+  enabled: boolean;
+};
+
+type JobStatusTableProps = {
+  environments: JobEnvironment[];
+};
+
+export function JobStatusTable({ environments }: JobStatusTableProps) {
+  return (
+    <Table fullWidth>
+      <TableHeader>
+        <TableRow>
+          <TableHeaderCell>Env</TableHeaderCell>
+          <TableHeaderCell>Last Run</TableHeaderCell>
+          <TableHeaderCell alignment="right">Version</TableHeaderCell>
+          <TableHeaderCell alignment="right">Status</TableHeaderCell>
+        </TableRow>
+      </TableHeader>
+      <TableBody>
+        {environments.map((environment, index) => (
+          <TableRow key={index}>
+            <TableCell>
+              <EnvironmentLabel environment={environment} />
+            </TableCell>
+            <TableCell>
+              {environment.lastRun ? <DateTime date={environment.lastRun} /> : "Never Run"}
+            </TableCell>
+            <TableCell alignment="right">{environment.version}</TableCell>
+            <TableCell alignment="right">
+              <ActiveBadge active={environment.enabled} />
+            </TableCell>
+          </TableRow>
+        ))}
+      </TableBody>
+    </Table>
+  );
+}

apps/webapp/app/components/code/CodeBlock.tsx

@@ -1,10 +1,9 @@
+import { Clipboard, ClipboardCheck } from "lucide-react";
 import type { Language, PrismTheme } from "prism-react-renderer";
 import Highlight, { defaultProps } from "prism-react-renderer";
 import { forwardRef, useCallback, useState } from "react";
 import { cn } from "~/utils/cn";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
-import { ClipboardDocumentCheckIcon, ClipboardIcon } from "@heroicons/react/24/solid";
-import { Clipboard, ClipboardCheck, ClipboardCheckIcon } from "lucide-react";
 
 //This is a fork of https://github.com/mantinedev/mantine/blob/master/src/mantine-prism/src/Prism/Prism.tsx
 //it didn't support highlighting lines by dimming the rest of the code, or animations on the highlighting
@@ -192,6 +191,9 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
       Array.from({ length: end - start + 1 }, (_, i) => start + i)
     );
 
+    // if there are more than 1000 lines, don't highlight
+    const shouldHighlight = lineCount <= 1000;
+
     return (
       <div
         className={cn("relative overflow-hidden rounded-md border border-slate-800", className)}
@@ -229,99 +231,113 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
           </TooltipProvider>
         )}
 
-        <Highlight {...defaultProps} theme={theme} code={code} language={language}>
-          {({
-            className: inheritedClassName,
-            style: inheritedStyle,
-            tokens,
-            getLineProps,
-            getTokenProps,
-          }) => (
-            <div
-              dir="ltr"
-              className="overflow-auto px-2 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-slate-700"
-              style={{
-                maxHeight,
-              }}
-            >
-              <pre
-                className={cn(
-                  "relative mr-2 font-mono text-xs leading-relaxed",
-                  inheritedClassName
-                )}
-                style={inheritedStyle}
+        {shouldHighlight ? (
+          <Highlight {...defaultProps} theme={theme} code={code} language={language}>
+            {({
+              className: inheritedClassName,
+              style: inheritedStyle,
+              tokens,
+              getLineProps,
+              getTokenProps,
+            }) => (
+              <div
                 dir="ltr"
+                className="overflow-auto px-2 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-slate-700"
+                style={{
+                  maxHeight,
+                }}
               >
-                {tokens
-                  .map((line, index) => {
-                    if (
-                      index === tokens.length - 1 &&
-                      line.length === 1 &&
-                      line[0].content === "\n"
-                    ) {
-                      return null;
-                    }
+                <pre
+                  className={cn(
+                    "relative mr-2 font-mono text-xs leading-relaxed",
+                    inheritedClassName
+                  )}
+                  style={inheritedStyle}
+                  dir="ltr"
+                >
+                  {tokens
+                    .map((line, index) => {
+                      if (
+                        index === tokens.length - 1 &&
+                        line.length === 1 &&
+                        line[0].content === "\n"
+                      ) {
+                        return null;
+                      }
 
-                    const lineNumber = index + 1;
-                    const lineProps = getLineProps({ line, key: index });
+                      const lineNumber = index + 1;
+                      const lineProps = getLineProps({ line, key: index });
 
-                    let hasAnyHighlights = highlightLines ? highlightLines.length > 0 : false;
+                      let hasAnyHighlights = highlightLines ? highlightLines.length > 0 : false;
 
-                    let shouldDim = hasAnyHighlights;
-                    if (hasAnyHighlights && highlightLines?.includes(lineNumber)) {
-                      shouldDim = false;
-                    }
+                      let shouldDim = hasAnyHighlights;
+                      if (hasAnyHighlights && highlightLines?.includes(lineNumber)) {
+                        shouldDim = false;
+                      }
 
-                    return (
-                      <div
-                        key={lineProps.key}
-                        {...lineProps}
-                        className={cn(
-                          "flex w-full justify-start transition-opacity duration-500",
-                          lineProps.className
-                        )}
-                        style={{
-                          opacity: shouldDim ? dimAmount : undefined,
-                          ...lineProps.style,
-                        }}
-                      >
-                        {showLineNumbers && (
-                          <div
-                            className={
-                              "mr-2 flex-none select-none text-right text-slate-500 transition-opacity duration-500"
-                            }
-                            style={{
-                              width: `calc(8 * ${maxLineWidth / 16}rem)`,
-                            }}
-                          >
-                            {lineNumber}
+                      return (
+                        <div
+                          key={lineProps.key}
+                          {...lineProps}
+                          className={cn(
+                            "flex w-full justify-start transition-opacity duration-500",
+                            lineProps.className
+                          )}
+                          style={{
+                            opacity: shouldDim ? dimAmount : undefined,
+                            ...lineProps.style,
+                          }}
+                        >
+                          {showLineNumbers && (
+                            <div
+                              className={
+                                "mr-2 flex-none select-none text-right text-slate-500 transition-opacity duration-500"
+                              }
+                              style={{
+                                width: `calc(8 * ${maxLineWidth / 16}rem)`,
+                              }}
+                            >
+                              {lineNumber}
+                            </div>
+                          )}
+
+                          <div className="flex-1">
+                            {line.map((token, key) => {
+                              const tokenProps = getTokenProps({ token, key });
+                              return (
+                                <span
+                                  key={tokenProps.key}
+                                  {...tokenProps}
+                                  style={{
+                                    color: tokenProps?.style?.color as string,
+                                    ...tokenProps.style,
+                                  }}
+                                />
+                              );
+                            })}
                           </div>
-                        )}
-
-                        <div className="flex-1">
-                          {line.map((token, key) => {
-                            const tokenProps = getTokenProps({ token, key });
-                            return (
-                              <span
-                                key={tokenProps.key}
-                                {...tokenProps}
-                                style={{
-                                  color: tokenProps?.style?.color as string,
-                                  ...tokenProps.style,
-                                }}
-                              />
-                            );
-                          })}
+                          <div className="w-4 flex-none" />
                         </div>
-                        <div className="w-4 flex-none" />
-                      </div>
-                    );
-                  })
-                  .filter(Boolean)}
-              </pre>
-            </div>
-          )}
-        </Highlight>
+                      );
+                    })
+                    .filter(Boolean)}
+                </pre>
+              </div>
+            )}
+          </Highlight>
+        ) : (
+          <div
+            dir="ltr"
+            className="overflow-auto px-2 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-slate-700"
+            style={{
+              maxHeight,
+            }}
+          >
+            <pre className="relative mr-2 p-2 font-mono text-xs leading-relaxed" dir="ltr">
+              {code}
+            </pre>
+          </div>
+        )}
       </div>
     );
   }

apps/webapp/app/components/environments/EnvironmentLabel.tsx

@@ -13,7 +13,7 @@ export function EnvironmentLabel({
   return (
     <span
       className={cn(
-        "flex h-4 items-center justify-center rounded-[2px] px-1 text-xxs font-medium uppercase tracking-wider text-midnight-900",
+        "inline-flex h-4 items-center justify-center rounded-[2px] px-1 text-xxs font-medium uppercase tracking-wider text-midnight-900",
         environmentColorClassName(environment),
         className
       )}

apps/webapp/app/components/helpContent/HelpContentText.tsx

@@ -414,6 +414,60 @@ export function HowToUseThisIntegration({ integration, help, integrationClient }
   );
 }
 
+export function HowToDisableAJob({
+  id,
+  name,
+  version,
+}: {
+  id: string;
+  name: string;
+  version: string;
+}) {
+  return (
+    <>
+      <Paragraph spacing>
+        To disable a job, you need to set the <InlineCode>enabled</InlineCode> property to{" "}
+        <InlineCode>false</InlineCode>.
+      </Paragraph>
+      <StepNumber
+        stepNumber="1"
+        title={
+          <>
+            Set <InlineCode>enabled</InlineCode> to <InlineCode>false</InlineCode>
+          </>
+        }
+      />
+      <StepContentContainer>
+        <CodeBlock
+          showLineNumbers={false}
+          className="mb-4"
+          code={`client.defineJob({
+  id: "${id}",
+  name: "${name}",
+  version: "${version}",
+  enabled: false,
+  // ...rest of your Job definition
+});`}
+        />
+      </StepContentContainer>
+      <StepNumber
+        stepNumber="2"
+        title={
+          <>
+            Run the <InlineCode>@trigger.dev/cli dev</InlineCode> command
+          </>
+        }
+      />
+      <StepContentContainer>
+        <Paragraph spacing>
+          If you aren't already running the <InlineCode>dev</InlineCode> command, run it now.
+        </Paragraph>
+        <TriggerDevCommand />
+      </StepContentContainer>
+    </>
+  );
+}
+
 export function HowToUseApiKeysAndEndpoints() {
   return (
     <>

apps/webapp/app/components/integrations/CustomHelp.tsx

@@ -25,8 +25,6 @@ client.defineJob({
     //wrap an SDK call in io.runTask so it's resumable and displays in logs
     const repo = await io.runTask(
       "Get repo",
-      //you can add metadata to the task to improve the display in the logs
-      { name: "Get repo", icon: "github" },
       async () => {
         //this is the regular GitHub SDK
         const response = await octokit.rest.repos.get({
@@ -34,7 +32,9 @@ client.defineJob({
           repo: "trigger.dev",
         });
         return response.data;
-      }
+      },
+      //you can add metadata to the task to improve the display in the logs
+      { name: "Get repo", icon: "github" }
     );
   },
 });
@@ -60,13 +60,13 @@ client.defineJob({
     //wrap anything in io.runTask so it's resumable and displays in logs
     const repo = await io.runTask(
       "Get org",
-      //you can add metadata to the task to improve the display in the logs
-      { name: "Get org", icon: "github" },
       async () => {
         //you can use fetch, axios, or any other library to make requests
         const response = await fetch('https://api.github.com/orgs/nodejs');
         return response.json();
-      }
+      },
+      //you can add metadata to the task to improve the display in the logs
+      { name: "Get org", icon: "github" }
     );
   },
 });

apps/webapp/app/components/jobs/DeleteJobModalContent.tsx

@@ -0,0 +1,109 @@
+import { RuntimeEnvironmentType } from "@trigger.dev/database";
+import { cn } from "~/utils/cn";
+import { JobStatusTable } from "../JobsStatusTable";
+import { Button } from "../primitives/Buttons";
+import { Header1, Header2 } from "../primitives/Headers";
+import { NamedIcon } from "../primitives/NamedIcon";
+import { Paragraph } from "../primitives/Paragraph";
+import { TextLink } from "../primitives/TextLink";
+import { useFetcher } from "@remix-run/react";
+import { Spinner } from "../primitives/Spinner";
+
+type JobEnvironment = {
+  type: RuntimeEnvironmentType;
+  lastRun?: Date;
+  version: string;
+  enabled: boolean;
+};
+
+type DeleteJobDialogContentProps = {
+  id: string;
+  title: string;
+  slug: string;
+  environments: JobEnvironment[];
+  redirectTo?: string;
+};
+
+export function DeleteJobDialogContent({
+  title,
+  slug,
+  environments,
+  id,
+  redirectTo,
+}: DeleteJobDialogContentProps) {
+  const canDelete = environments.every((environment) => !environment.enabled);
+  const fetcher = useFetcher();
+
+  const isLoading =
+    fetcher.state === "submitting" ||
+    (fetcher.state === "loading" && fetcher.formMethod === "DELETE");
+
+  return (
+    <div className="flex w-full flex-col items-center gap-y-6">
+      <div className="flex flex-col items-center justify-center gap-y-2">
+        <Header1>{title}</Header1>
+        <Paragraph variant="small">ID: {slug}</Paragraph>
+      </div>
+      <JobStatusTable environments={environments} />
+
+      <Header2
+        className={cn(
+          canDelete ? "border-rose-500 bg-rose-500/10" : "border-amber-500 bg-amber-500/10",
+          "rounded border px-3.5 py-2 text-center text-bright"
+        )}
+      >
+        {canDelete
+          ? "Are you sure you want to delete this Job?"
+          : "You can't delete this Job until all env are disabled"}
+      </Header2>
+      <Paragraph variant="small" className="px-6 text-center">
+        {canDelete ? (
+          <>
+            This will permanently delete the Job <span className="strong text-bright">{title}</span>
+            . This includes the deletion of all Run history. This cannot be undone.
+          </>
+        ) : (
+          <>
+            This Job is still active in an environment. You need to disable it in your Job code
+            first before it can be deleted.{" "}
+            <TextLink to="https://trigger.dev/docs/documentation/guides/jobs/managing#disabling-jobs">
+              Learn how to disable a Job
+            </TextLink>
+            .
+          </>
+        )}
+      </Paragraph>
+      {canDelete ? (
+        <fetcher.Form
+          method="delete"
+          action={`/resources/jobs/${id}${redirectTo ? `?redirectTo=${redirectTo}` : ""}`}
+          className="w-full"
+        >
+          <Button variant="danger/large" fullWidth>
+            {isLoading ? (
+              <Spinner color="white" />
+            ) : (
+              <>
+                <NamedIcon
+                  name="trash-can"
+                  className="mr-1.5 h-4 w-4 text-bright transition group-hover:text-bright"
+                />
+                Delete this Job
+              </>
+            )}
+          </Button>
+        </fetcher.Form>
+      ) : (
+        <Button variant="danger/large" fullWidth disabled>
+          <>
+            <NamedIcon
+              name="trash-can"
+              className="mr-1.5 h-4 w-4 text-bright transition group-hover:text-bright"
+            />
+            Delete this Job
+          </>
+        </Button>
+      )}
+    </div>
+  );
+}

apps/webapp/app/components/jobs/JobStatusBadge.tsx

@@ -0,0 +1,29 @@
+import { ActiveBadge, MissingIntegrationBadge, NewBadge } from "../ActiveBadge";
+
+type JobStatusBadgeProps = {
+  enabled: boolean;
+  hasIntegrationsRequiringAction: boolean;
+  hasRuns: boolean;
+  badgeSize?: "small" | "normal";
+};
+
+export function JobStatusBadge({
+  enabled,
+  hasIntegrationsRequiringAction,
+  hasRuns,
+  badgeSize = "normal",
+}: JobStatusBadgeProps) {
+  if (!enabled) {
+    return <ActiveBadge active={false} badgeSize={badgeSize} />;
+  }
+
+  if (hasIntegrationsRequiringAction) {
+    return <MissingIntegrationBadge badgeSize={badgeSize} />;
+  }
+
+  if (!hasRuns) {
+    return <NewBadge badgeSize={badgeSize} />;
+  }
+
+  return <ActiveBadge active={true} badgeSize={badgeSize} />;
+}

apps/webapp/app/components/jobs/JobsTable.tsx

@@ -1,26 +1,29 @@
-import { jobPath } from "~/utils/pathBuilder";
+import { ProjectJob } from "~/hooks/useJobs";
+import { useOrganization } from "~/hooks/useOrganizations";
+import { useProject } from "~/hooks/useProject";
+import { JobRunStatus } from "~/models/job.server";
+import { jobPath, jobTestPath } from "~/utils/pathBuilder";
+import { Button } from "../primitives/Buttons";
 import { DateTime } from "../primitives/DateTime";
+import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog";
 import { LabelValueStack } from "../primitives/LabelValueStack";
 import { NamedIcon } from "../primitives/NamedIcon";
 import { Paragraph } from "../primitives/Paragraph";
+import { PopoverMenuItem } from "../primitives/Popover";
 import {
   Table,
   TableBlankRow,
   TableBody,
   TableCell,
-  TableCellChevron,
+  TableCellMenu,
   TableHeader,
   TableHeaderCell,
   TableRow,
 } from "../primitives/Table";
 import { SimpleTooltip } from "../primitives/Tooltip";
 import { runStatusTitle } from "../runs/RunStatuses";
-import { ProjectJob } from "~/hooks/useJobs";
-import { useProject } from "~/hooks/useProject";
-import { useOrganization } from "~/hooks/useOrganizations";
-import { JobRunStatus } from "~/models/job.server";
-import { cn } from "~/utils/cn";
-import { Badge } from "../primitives/Badge";
+import { DeleteJobDialogContent } from "./DeleteJobModalContent";
+import { JobStatusBadge } from "./JobStatusBadge";
 
 export function JobsTable({ jobs, noResultsText }: { jobs: ProjectJob[]; noResultsText: string }) {
   const organization = useOrganization();
@@ -35,6 +38,7 @@ export function JobsTable({ jobs, noResultsText }: { jobs: ProjectJob[]; noResul
           <TableHeaderCell>Integrations</TableHeaderCell>
           <TableHeaderCell>Properties</TableHeaderCell>
           <TableHeaderCell>Last run</TableHeaderCell>
+          <TableHeaderCell>Status</TableHeaderCell>
           <TableHeaderCell hiddenLabel>Go to page</TableHeaderCell>
         </TableRow>
       </TableHeader>
@@ -43,13 +47,7 @@ export function JobsTable({ jobs, noResultsText }: { jobs: ProjectJob[]; noResul
           jobs.map((job) => {
             const path = jobPath(organization, project, job);
             return (
-              <TableRow
-                key={job.id}
-                className={cn(
-                  (job.hasIntegrationsRequiringAction && "bg-rose-500/20") ||
-                    (job.lastRun === undefined && "bg-green-500/20")
-                )}
-              >
+              <TableRow key={job.id} className="group">
                 <TableCell to={path}>
                   <span className="flex items-center gap-2">
                     <NamedIcon name={job.event.icon} className="h-8 w-8" />
@@ -145,13 +143,39 @@ export function JobsTable({ jobs, noResultsText }: { jobs: ProjectJob[]; noResul
                     <LabelValueStack label={"Never run"} value={"–"} />
                   )}
                 </TableCell>
-                <TableCellChevron to={path}>
-                  {job.lastRun === undefined && (
-                    <Badge className="mr-4" variant="green">
-                      New Job!
-                    </Badge>
-                  )}
-                </TableCellChevron>
+                <TableCell to={path}>
+                  <JobStatusBadge
+                    enabled={job.status === "ACTIVE"}
+                    hasIntegrationsRequiringAction={job.hasIntegrationsRequiringAction}
+                    hasRuns={job.lastRun !== undefined}
+                  />
+                </TableCell>
+                <TableCellMenu isSticky>
+                  <PopoverMenuItem to={path} title="View Job" icon="eye" />
+                  <PopoverMenuItem
+                    to={jobTestPath(organization, project, job)}
+                    title="Test Job"
+                    icon="beaker"
+                  />
+
+                  <Dialog>
+                    <DialogTrigger asChild>
+                      <Button variant="menu-item" LeadingIcon="trash-can">
+                        Delete Job
+                      </Button>
+                    </DialogTrigger>
+                    <DialogContent>
+                      <DialogHeader>
+                        <DeleteJobDialogContent
+                          id={job.id}
+                          title={job.title}
+                          slug={job.slug}
+                          environments={job.environments}
+                        />
+                      </DialogHeader>
+                    </DialogContent>
+                  </Dialog>
+                </TableCellMenu>
               </TableRow>
             );
           })

apps/webapp/app/components/navigation/NavBar.tsx

@@ -11,7 +11,7 @@ import { docsRoot } from "~/utils/pathBuilder";
 
 export function NavBar() {
   return (
-    <div className="z-50 flex w-full items-center justify-between gap-2 border-b border-divide py-1 pl-1 pr-2.5">
+    <div className="z-50 flex w-full items-center justify-between gap-2 border-b border-uiBorder py-1 pl-1 pr-2.5">
       <div className="flex gap-0.5">
         <Link to="/" className="p-2">
           <LogoIcon className="h-5 w-5" />

apps/webapp/app/components/navigation/ProjectSideMenu.tsx

@@ -58,7 +58,7 @@ export function ProjectSideMenu() {
       variants={menuVariants}
       initial={isCollapsed ? "collapsed" : "expanded"}
       className={cn(
-        "flex h-full flex-col justify-between overflow-hidden border-r border-slate-850 p-1 transition duration-300 ease-in-out"
+        "flex h-full flex-col justify-between overflow-hidden border-r border-uiBorder p-1 transition duration-300 ease-in-out"
       )}
     >
       <div className="flex flex-col gap-1">

apps/webapp/app/components/primitives/Badge.tsx

@@ -6,8 +6,6 @@ const variants = {
     "grid place-items-center rounded-full px-2 h-5 tracking-wider text-xxs bg-slate-700 text-bright uppercase whitespace-nowrap",
   outline:
     "grid place-items-center rounded-sm px-1 h-5 tracking-wider text-xxs border border-dimmed text-dimmed uppercase whitespace-nowrap",
-  green:
-    "grid place-items-center rounded-sm px-1.5 h-5 tracking-wider outline-offset-1 outline outline-1 outline-green-600 text-xxs bg-green-500 text-slate-900 uppercase whitespace-nowrap",
 };
 
 type BadgeProps = React.HTMLAttributes<HTMLDivElement> & {

apps/webapp/app/components/primitives/Buttons.tsx

@@ -87,6 +87,14 @@ const variant = {
     shortcutVariant: undefined,
     shortcut: undefined,
   },
+  "danger/large": {
+    textColor: "text-bright group-hover:text-white transition group-disabled:text-bright/50",
+    button:
+      "h-10 px-2 text-md bg-rose-600 group-hover:bg-rose-500 group-disabled:opacity-50 group-disabled:group-hover:bg-rose-600",
+    icon: "h-5",
+    shortcutVariant: "medium" as const,
+    shortcut: "ml-1.5 -mr-0.5 border-bright/40 text-bright group-hover:border-bright/60",
+  },
   "menu-item": {
     textColor: "text-bright",
     button:

apps/webapp/app/components/primitives/Dialog.tsx

@@ -2,8 +2,9 @@
 
 import * as React from "react";
 import * as DialogPrimitive from "@radix-ui/react-dialog";
-import { X } from "lucide-react";
 import { cn } from "~/utils/cn";
+import { XMarkIcon } from "@heroicons/react/24/solid";
+import { ShortcutKey } from "./ShortcutKey";
 
 const Dialog = DialogPrimitive.Root;
 
@@ -25,7 +26,7 @@ const DialogOverlay = React.forwardRef<
   <DialogPrimitive.Overlay
     ref={ref}
     className={cn(
-      "data-[state=closed]:animate-out data-[state=closed]:fade-out data-[state=open]:fade-in fixed inset-0 z-50 bg-background/80 backdrop-blur-sm transition-all duration-100",
+      "fixed inset-0 z-50 bg-background/80 backdrop-blur-sm transition-all duration-100 data-[state=closed]:animate-out data-[state=closed]:fade-out data-[state=open]:fade-in",
       className
     )}
     {...props}
@@ -42,15 +43,29 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 data-[state=open]:sm:slide-in-from-bottom-0 fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background p-6 shadow-lg animate-in sm:max-w-lg sm:rounded-lg sm:zoom-in-90",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-midnight-800 p-6 pt-11 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}
     >
+      <hr className="-ml-6 w-[calc(100%_+_3rem)]" />
       {children}
-      <DialogPrimitive.Close className="data-[state=open]:bg-accent data-[state=open]:text-muted-foreground absolute right-4 top-4 rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none">
-        <X className="h-4 w-4" />
-        <span className="sr-only">Close</span>
+      <DialogPrimitive.Close className="absolute right-3 top-3 rounded-sm opacity-70 ring-offset-background transition-opacity data-[state=open]:bg-accent data-[state=open]:text-muted-foreground hover:opacity-100 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none">
+        <div className="flex gap-x-2">
+          <ShortcutKey
+            shortcut={{
+              windows: {
+                key: "esc",
+              },
+              mac: {
+                key: "esc",
+              },
+            }}
+            variant={"small"}
+          />
+          <XMarkIcon className="h-5 w-5" />
+          <span className="sr-only">Close</span>
+        </div>
       </DialogPrimitive.Close>
     </DialogPrimitive.Content>
   </DialogPortal>

apps/webapp/app/components/primitives/FormTitle.tsx

@@ -21,7 +21,7 @@ export function FormTitle({
     <div
       className={cn(
         "mb-4 flex flex-col gap-3 pb-4",
-        divide ? "border-b border-slate-800" : "",
+        divide ? "border-b border-uiBorder" : "",
         className
       )}
     >

apps/webapp/app/components/primitives/NamedIcon.tsx

@@ -27,6 +27,7 @@ import {
   Cog8ToothIcon,
   CreditCardIcon,
   EnvelopeIcon,
+  EyeIcon,
   FingerPrintIcon,
   FlagIcon,
   FolderIcon,
@@ -41,6 +42,7 @@ import {
   QrCodeIcon,
   SquaresPlusIcon,
   StarIcon,
+  TrashIcon,
   UserCircleIcon,
   UserGroupIcon,
   UserIcon,
@@ -50,7 +52,7 @@ import {
   XMarkIcon,
 } from "@heroicons/react/24/solid";
 import { CompanyIcon, hasIcon } from "@trigger.dev/companyicons";
-import { HourglassIcon } from "lucide-react";
+import { ActivityIcon, HourglassIcon } from "lucide-react";
 import { DynamicTriggerIcon } from "~/assets/icons/DynamicTriggerIcon";
 import { ErrorIcon } from "~/assets/icons/ErrorIcon";
 import { ScheduleIcon } from "~/assets/icons/ScheduleIcon";
@@ -100,6 +102,7 @@ const icons = {
   folder: (className: string) => <FolderIcon className={cn("text-indigo-600", className)} />,
   envelope: (className: string) => <EnvelopeIcon className={cn("text-cyan-500", className)} />,
   environment: (className: string) => <KeyIcon className={cn("text-yellow-500", className)} />,
+  eye: (className: string) => <EyeIcon className={cn("text-blue-500", className)} />,
   globe: (className: string) => <GlobeAltIcon className={cn("text-fuchsia-600", className)} />,
   "hand-raised": (className: string) => (
     <HandRaisedIcon className={cn("text-amber-400", className)} />
@@ -129,6 +132,7 @@ const icons = {
     <PlusSmallIcon className={cn("text-green-600", className)} />
   ),
   property: (className: string) => <Cog8ToothIcon className={cn("text-slate-600", className)} />,
+  pulse: (className: string) => <ActivityIcon className={cn("text-green-600", className)} />,
   "qr-code": (className: string) => <QrCodeIcon className={cn("text-amber-400", className)} />,
   refresh: (className: string) => <ArrowPathIcon className={cn("text-bright", className)} />,
   sapling: (className: string) => <SaplingIcon className={cn("text-green-500", className)} />,
@@ -139,6 +143,7 @@ const icons = {
   star: (className: string) => <StarIcon className={cn("text-yellow-500", className)} />,
   stop: (className: string) => <StopIcon className={cn("text-rose-500", className)} />,
   team: (className: string) => <UserGroupIcon className={cn("text-blue-500", className)} />,
+  "trash-can": (className: string) => <TrashIcon className={cn("text-rose-500", className)} />,
   tree: (className: string) => <OneTreeIcon className={cn("text-green-500", className)} />,
   trees: (className: string) => <TwoTreesIcon className={cn("text-green-500", className)} />,
   trigger: (className: string) => <BoltIcon className={cn("text-orange-500", className)} />,

apps/webapp/app/components/primitives/PageHeader.tsx

@@ -13,7 +13,7 @@ type WithChildren = {
 
 export function PageHeader({ children, hideBorder }: WithChildren & { hideBorder?: boolean }) {
   return (
-    <div className={cn("mx-4 pt-4", hideBorder ? "" : "border-b border-slate-800 pb-4")}>
+    <div className={cn("mx-4 pt-4", hideBorder ? "" : "border-b border-uiBorder pb-4")}>
       {children}
     </div>
   );

apps/webapp/app/components/primitives/Popover.tsx

@@ -4,9 +4,8 @@ import * as React from "react";
 import * as PopoverPrimitive from "@radix-ui/react-popover";
 import { cn } from "~/utils/cn";
 import { Paragraph } from "./Paragraph";
-import { ChevronDownIcon } from "@heroicons/react/24/solid";
+import { ChevronDownIcon, EllipsisVerticalIcon } from "@heroicons/react/24/solid";
 import { LinkButton } from "./Buttons";
-import { IconNames } from "./NamedIcon";
 
 const Popover = PopoverPrimitive.Root;
 
@@ -91,6 +90,24 @@ function PopoverArrowTrigger({
   );
 }
 
+function PopoverVerticalEllipseTrigger({
+  isOpen,
+  className,
+  ...props
+}: { isOpen?: boolean } & React.ComponentPropsWithoutRef<typeof PopoverTrigger>) {
+  return (
+    <PopoverTrigger
+      {...props}
+      className={cn(
+        "group flex items-center justify-end gap-1 rounded px-1.5 py-1.5 text-dimmed transition hover:bg-slate-750 hover:text-bright",
+        className
+      )}
+    >
+      <EllipsisVerticalIcon className={cn("h-5 w-5 transition group-hover:text-bright")} />
+    </PopoverTrigger>
+  );
+}
+
 export {
   Popover,
   PopoverTrigger,
@@ -98,4 +115,5 @@ export {
   PopoverSectionHeader,
   PopoverArrowTrigger,
   PopoverMenuItem,
+  PopoverVerticalEllipseTrigger,
 };

apps/webapp/app/components/primitives/Sheet.tsx

@@ -51,7 +51,7 @@ const SheetOverlay = React.forwardRef<
 SheetOverlay.displayName = SheetPrimitive.Overlay.displayName;
 
 const sheetVariants = cva(
-  "fixed z-50 scale-100 gap-4 bg-midnight-900 shadow-lg shadow-white/10 opacity-100 border-l border-y border-slate-800",
+  "fixed z-50 scale-100 gap-4 bg-midnight-900 shadow-lg shadow-white/10 opacity-100 border-l border-uiBorder",
   {
     variants: {
       position: {
@@ -154,7 +154,7 @@ const SheetContent = React.forwardRef<
       {...props}
     >
       <div className="grid max-h-full grid-rows-[2.75rem_1fr] overflow-hidden">
-        <div className="flex items-center gap-2 border-b border-slate-800 p-2">
+        <div className="flex items-center gap-2 border-b border-uiBorder p-2">
           <SheetPrimitive.Close className="rounded-sm p-1 transition hover:bg-slate-800 disabled:pointer-events-none">
             <NamedIcon name="close" className="h-4 w-4" />
             <span className="sr-only">Close</span>
@@ -181,7 +181,7 @@ export const SheetBody = ({ className, ...props }: React.HTMLAttributes<HTMLDivE
 export const SheetHeader = ({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) => (
   <div
     className={cn(
-      "mx-4 flex shrink-0 items-center gap-4 border-b border-slate-800 py-3.5",
+      "mx-4 flex shrink-0 items-center gap-4 border-b border-uiBorder py-3.5",
       className
     )}
     {...props}
@@ -194,7 +194,7 @@ export const SheetFooter = ({
   ...props
 }: React.HTMLAttributes<HTMLDivElement>) => (
   <div className={cn("shrink-0", className)} {...props}>
-    <div className="mx-4 border-t border-slate-800 py-3">{children}</div>
+    <div className="mx-4 border-t border-uiBorder py-3">{children}</div>
   </div>
 );
 

apps/webapp/app/components/primitives/Table.tsx

@@ -1,25 +1,27 @@
 import { ChevronRightIcon } from "@heroicons/react/24/solid";
 import { Link } from "@remix-run/react";
-import { ReactNode, forwardRef } from "react";
+import { ReactNode, forwardRef, useState } from "react";
 import { cn } from "~/utils/cn";
-import { Badge } from "./Badge";
+import { Popover, PopoverContent, PopoverVerticalEllipseTrigger } from "./Popover";
 
 type TableProps = {
   containerClassName?: string;
   className?: string;
   children: ReactNode;
+  fullWidth?: boolean;
 };
 
 export const Table = forwardRef<HTMLTableElement, TableProps>(
-  ({ className, containerClassName, children }, ref) => {
+  ({ className, containerClassName, children, fullWidth }, ref) => {
     return (
       <div
         className={cn(
-          "overflow-x-auto whitespace-nowrap rounded-md border border-slate-900 scrollbar-thin scrollbar-track-midnight-850 scrollbar-thumb-slate-700",
-          containerClassName
+          "overflow-x-auto whitespace-nowrap rounded-md border border-uiBorder scrollbar-thin scrollbar-track-midnight-850 scrollbar-thumb-slate-700",
+          containerClassName,
+          fullWidth && "w-full"
         )}
       >
-        <table ref={ref} className={cn("w-full divide-y bg-midnight-850", className)}>
+        <table ref={ref} className={cn("w-full divide-y", className)}>
           {children}
         </table>
       </div>
@@ -37,7 +39,7 @@ export const TableHeader = forwardRef<HTMLTableSectionElement, TableHeaderProps>
     return (
       <thead
         ref={ref}
-        className={cn("rounded-t-md", "relative divide-y divide-slate-850", className)}
+        className={cn("rounded-t-md", "relative divide-y divide-uiBorder bg-slate-850", className)}
       >
         {children}
       </thead>
@@ -53,7 +55,7 @@ type TableBodyProps = {
 export const TableBody = forwardRef<HTMLTableSectionElement, TableBodyProps>(
   ({ className, children }, ref) => {
     return (
-      <tbody ref={ref} className={cn("relative divide-y divide-slate-850", className)}>
+      <tbody ref={ref} className={cn("relative divide-y divide-uiBorder", className)}>
         {children}
       </tbody>
     );
@@ -63,12 +65,13 @@ export const TableBody = forwardRef<HTMLTableSectionElement, TableBodyProps>(
 type TableRowProps = {
   className?: string;
   children: ReactNode;
+  disabled?: boolean;
 };
 
 export const TableRow = forwardRef<HTMLTableRowElement, TableRowProps>(
-  ({ className, children }, ref) => {
+  ({ className, disabled, children }, ref) => {
     return (
-      <tr ref={ref} className={cn("group w-full", className)}>
+      <tr ref={ref} className={cn(disabled && "opacity-50", "group w-full", className)}>
         {children}
       </tr>
     );
@@ -103,7 +106,7 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
         ref={ref}
         scope="col"
         className={cn(
-          "px-4 py-3 align-middle text-xs font-semibold uppercase text-slate-400",
+          "px-4 py-3 align-middle text-xs font-normal uppercase tracking-wider text-dimmed",
           alignmentClassName,
           className
         )}
@@ -118,10 +121,11 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
 type TableCellProps = TableCellBasicProps & {
   to?: string;
   onClick?: (event: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void;
+  hasAction?: boolean;
 };
 
 export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
-  ({ className, alignment = "left", children, colSpan, to, onClick }, ref) => {
+  ({ className, alignment = "left", children, colSpan, to, onClick, hasAction = false }, ref) => {
     let alignmentClassName = "text-left";
     switch (alignment) {
       case "center":
@@ -133,7 +137,7 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
     }
 
     const flexClasses = cn(
-      "flex w-full whitespace-nowrap px-4 py-3 text-xs text-slate-400",
+      "flex w-full whitespace-nowrap px-4 py-3 text-xs text-dimmed",
       alignment === "left"
         ? "justify-start text-left"
         : alignment === "center"
@@ -145,8 +149,10 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
       <td
         ref={ref}
         className={cn(
-          "text-xs text-slate-400 transition group-hover:bg-slate-850/50",
-          to || onClick ? "cursor-pointer" : "px-4 py-3 align-middle",
+          "text-xs text-slate-400",
+          to || onClick || hasAction
+            ? "cursor-pointer group-hover:bg-slate-900"
+            : "px-4 py-3 align-middle",
           !to && !onClick && alignmentClassName,
           className
         )}
@@ -168,23 +174,64 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
   }
 );
 
+const stickyStyles =
+  "sticky right-0 z-10 w-[2.8rem] min-w-[2.8rem] bg-background before:absolute before:pointer-events-none before:-left-8 before:top-0 before:h-full before:min-w-[2rem] before:bg-gradient-to-r before:from-transparent before:to-background before:content-[''] group-hover:before:to-slate-900";
+
 export const TableCellChevron = forwardRef<
   HTMLTableCellElement,
   {
     className?: string;
     to?: string;
     children?: ReactNode;
+    isSticky?: boolean;
     onClick?: (event: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void;
   }
->(({ className, to, children, onClick }, ref) => {
+>(({ className, to, children, isSticky, onClick }, ref) => {
   return (
-    <TableCell className={className} to={to} onClick={onClick} ref={ref} alignment="right">
+    <TableCell
+      className={cn(isSticky && stickyStyles, className)}
+      to={to}
+      onClick={onClick}
+      ref={ref}
+      alignment="right"
+    >
       {children}
       <ChevronRightIcon className="h-4 w-4 text-dimmed transition group-hover:text-bright" />
     </TableCell>
   );
 });
 
+export const TableCellMenu = forwardRef<
+  HTMLTableCellElement,
+  {
+    className?: string;
+    children?: ReactNode;
+    isSticky?: boolean;
+    onClick?: (event: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void;
+  }
+>(({ className, children, isSticky, onClick }, ref) => {
+  const [isOpen, setIsOpen] = useState(false);
+  return (
+    <TableCell
+      className={cn(isSticky && stickyStyles, className)}
+      onClick={onClick}
+      ref={ref}
+      alignment="right"
+      hasAction={true}
+    >
+      <Popover onOpenChange={(open) => setIsOpen(open)}>
+        <PopoverVerticalEllipseTrigger isOpen={isOpen} />
+        <PopoverContent
+          className="w-fit max-w-[10rem] overflow-y-auto p-0 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-slate-700"
+          align="end"
+        >
+          <div className="flex flex-col gap-1 p-1">{children}</div>
+        </PopoverContent>
+      </Popover>
+    </TableCell>
+  );
+});
+
 type TableBlankRowProps = {
   className?: string;
   colSpan: number;

apps/webapp/app/components/primitives/Tabs.tsx

@@ -12,7 +12,7 @@ export type TabsProps = {
 
 export function Tabs({ tabs, className }: TabsProps) {
   return (
-    <div className={cn(`flex flex-row gap-x-6 border-b border-slate-800`, className)}>
+    <div className={cn(`flex flex-row gap-x-6 border-b border-uiBorder`, className)}>
       {tabs.map((tab, index) => (
         <NavLink key={index} to={tab.to} className="group flex flex-col items-center pt-1" end>
           {({ isActive, isPending }) => (

apps/webapp/app/components/run/RunCompletedDetail.tsx

@@ -54,7 +54,7 @@ export function RunCompletedDetail({ run }: { run: MatchedRun }) {
         <RunPanelDivider />
         {run.error && <RunPanelError text={run.error.message} stackTrace={run.error.stack} />}
         {run.output ? (
-          <CodeBlock language="json" code={run.output} />
+          <CodeBlock language="json" code={run.output} maxLines={36} />
         ) : (
           run.output === null && <Paragraph variant="small">This run returned nothing</Paragraph>
         )}

apps/webapp/app/components/run/RunOverview.tsx

@@ -273,6 +273,7 @@ function BlankTasks({
   basicStatus: RunBasicStatus;
 }) {
   switch (basicStatus) {
+    default:
     case "COMPLETED":
       return <Paragraph variant="small">There were no tasks for this run.</Paragraph>;
     case "FAILED":
@@ -288,8 +289,6 @@ function BlankTasks({
           <TaskCardSkeleton />
         </div>
       );
-    default:
-      return <Paragraph variant="small">There were no tasks for this run.</Paragraph>;
   }
 }
 

apps/webapp/app/components/run/TaskDetail.tsx

@@ -1,4 +1,3 @@
-import { DetailedTask } from "~/presenters/TaskDetailsPresenter.server";
 import {
   RunPanel,
   RunPanelBody,
@@ -29,22 +28,16 @@ import {
 } from "../primitives/Table";
 import { TaskAttemptStatusLabel } from "./TaskAttemptStatus";
 import { TaskStatusIcon } from "./TaskStatus";
+import { ClientOnly } from "remix-utils";
+import { Spinner } from "../primitives/Spinner";
+import type { DetailedTask } from "~/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route";
 
 export function TaskDetail({ task }: { task: DetailedTask }) {
-  const {
-    name,
-    description,
-    icon,
-    startedAt,
-    completedAt,
-    status,
-    delayUntil,
-    params,
-    properties,
-    output,
-    style,
-    attempts,
-  } = task;
+  const { name, description, icon, status, params, properties, output, style, attempts } = task;
+
+  const startedAt = task.startedAt ? new Date(task.startedAt) : undefined;
+  const completedAt = task.completedAt ? new Date(task.completedAt) : undefined;
+  const delayUntil = task.delayUntil ? new Date(task.delayUntil) : undefined;
 
   return (
     <RunPanel selected={false}>
@@ -150,7 +143,9 @@ export function TaskDetail({ task }: { task: DetailedTask }) {
         <div className="mt-4 flex flex-col gap-2">
           <Header3>Output</Header3>
           {output ? (
-            <CodeBlock code={JSON.stringify(output, null, 2)} />
+            <ClientOnly fallback={<Spinner />}>
+              {() => <CodeBlock code={output} maxLines={35} />}
+            </ClientOnly>
           ) : (
             <Paragraph variant="small">No output</Paragraph>
           )}

apps/webapp/app/components/runs/RunsTable.tsx

@@ -109,7 +109,7 @@ export function RunsTable({
                 <TableCell to={path}>
                   {run.createdAt ? <DateTime date={run.createdAt} /> : "–"}
                 </TableCell>
-                <TableCellChevron to={path} />
+                <TableCellChevron to={path} isSticky />
               </TableRow>
             );
           })

apps/webapp/app/components/stories/Badges.stories.tsx

@@ -20,7 +20,6 @@ function BadgesExample() {
     <div className="flex flex-col items-start gap-y-8 p-8">
       <Badge>Default</Badge>
       <Badge variant="outline">Outline</Badge>
-      <Badge variant="green">Green</Badge>
     </div>
   );
 }

apps/webapp/app/components/stories/Button.stories.tsx

@@ -253,13 +253,11 @@ function ButtonList({ primary }: { primary: string }) {
       <Header1 className="mb-2 mt-8">Large buttons</Header1>
       <div className="grid grid-cols-1 gap-8 border-b border-slate-700 pb-8">
         <div className="flex flex-col gap-2">
-          <div className="flex flex-col items-start">
+          <div className="flex flex-col items-start gap-2">
             <Button variant="primary/large" fullWidth>
               <NamedIcon name={"github"} className={"mr-1.5 h-4 w-4"} />
               Continue with GitHub
             </Button>
-          </div>
-          <div className="flex flex-col items-start">
             <Button variant="secondary/large" fullWidth>
               <NamedIcon
                 name={"envelope"}
@@ -267,6 +265,13 @@ function ButtonList({ primary }: { primary: string }) {
               />
               Continue with Email
             </Button>
+            <Button variant="danger/large" fullWidth>
+              <NamedIcon
+                name={"trash-can"}
+                className={"mr-1.5 h-4 w-4 text-bright transition group-hover:text-bright"}
+              />
+              This is a delete button
+            </Button>
           </div>
         </div>
       </div>

apps/webapp/app/consts.ts

@@ -1,6 +1,7 @@
 export const LIVE_ENVIRONMENT = "live";
 export const DEV_ENVIRONMENT = "development";
 export const MAX_LIVE_PROJECTS = 1;
-export const DEFAULT_MAX_CONCURRENT_RUNS = 10000;
+export const DEFAULT_MAX_CONCURRENT_RUNS = 10;
+export const MAX_CONCURRENT_RUNS_LIMIT = 20;
 export const PREPROCESS_RETRY_LIMIT = 2;
 export const EXECUTE_JOB_RETRY_LIMIT = 10;

apps/webapp/app/db.server.ts

@@ -2,6 +2,7 @@ import { PrismaClient, Prisma } from "@trigger.dev/database";
 import invariant from "tiny-invariant";
 import { z } from "zod";
 import { logger } from "./services/logger.server";
+import { env } from "./env.server";
 
 export type PrismaTransactionClient = Omit<
   PrismaClient,
@@ -29,6 +30,8 @@ export type PrismaTransactionOptions = {
 
   /**  Sets the transaction isolation level. By default this is set to the value currently configured in your database. */
   isolationLevel?: Prisma.TransactionIsolationLevel;
+
+  rethrowPrismaErrors?: boolean;
 };
 
 export async function $transaction<R>(
@@ -52,6 +55,10 @@ export async function $transaction<R>(
         name: error.name,
       });
 
+      if (options?.rethrowPrismaErrors) {
+        throw error;
+      }
+
       return;
     }
 
@@ -84,8 +91,16 @@ function getClient() {
   const { DATABASE_URL } = process.env;
   invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");
 
+  const databaseUrl = new URL(DATABASE_URL);
+
+  // We need to add the connection_limit and pool_timeout query params to the url, in a way that works if the DATABASE_URL already has query params
+  const query = databaseUrl.searchParams;
+  query.set("connection_limit", env.DATABASE_CONNECTION_LIMIT.toString());
+  query.set("pool_timeout", env.DATABASE_POOL_TIMEOUT.toString());
+  databaseUrl.search = query.toString();
+
   // Remove the username:password in the url and print that to the console
-  const urlWithoutCredentials = new URL(DATABASE_URL);
+  const urlWithoutCredentials = new URL(databaseUrl.href);
   urlWithoutCredentials.password = "";
 
   console.log(`🔌 setting up prisma client to ${urlWithoutCredentials.toString()}`);
@@ -93,8 +108,7 @@ function getClient() {
   const client = new PrismaClient({
     datasources: {
       db: {
-        url: DATABASE_URL,
-        // We can't set directUrl here, and we don't have to
+        url: databaseUrl.href,
       },
     },
     log: [

apps/webapp/app/entry.server.tsx

@@ -74,7 +74,7 @@ function serveTheBots(
       {
         // Use onAllReady to wait for the entire document to be ready
         onAllReady() {
-          responseHeaders.set("Content-Type", "text/html");
+          responseHeaders.set("Content-Type", "text/html; charset=utf-8");
           let body = new PassThrough();
           pipe(body);
           resolve(
@@ -114,7 +114,7 @@ function serveBrowsers(
         // use onShellReady to wait until a suspense boundary is triggered
         onShellReady() {
           shellReady = true;
-          responseHeaders.set("Content-Type", "text/html");
+          responseHeaders.set("Content-Type", "text/html; charset=utf-8");
           let body = new PassThrough();
           pipe(body);
           resolve(

apps/webapp/app/env.server.ts

@@ -4,6 +4,8 @@ import { SecretStoreOptionsSchema } from "./services/secrets/secretStore.server"
 const EnvironmentSchema = z.object({
   NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]),
   DATABASE_URL: z.string(),
+  DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10),
+  DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60),
   DIRECT_URL: z.string(),
   SESSION_SECRET: z.string(),
   MAGIC_LINK_SECRET: z.string(),
@@ -31,6 +33,21 @@ const EnvironmentSchema = z.object({
   RESEND_API_KEY: z.string().optional(),
   PLAIN_API_KEY: z.string().optional(),
   RUNTIME_PLATFORM: z.enum(["docker-compose", "ecs", "local"]).default("local"),
+  WORKER_SCHEMA: z.string().default("graphile_worker"),
+  WORKER_CONCURRENCY: z.coerce.number().int().default(10),
+  WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
+  EXECUTION_WORKER_CONCURRENCY: z.coerce.number().int().default(10),
+  EXECUTION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
+  WORKER_ENABLED: z.string().default("true"),
+  EXECUTION_WORKER_ENABLED: z.string().default("true"),
+  // Docker Registry
+  DOCKER_REGISTRY_HOST: z.string().optional(),
+  DOCKER_REGISTRY_USERNAME: z.string().optional(),
+  DOCKER_REGISTRY_PASSWORD: z.string().optional(),
+  // Fly Background Task Provider
+  FLY_IO_API_TOKEN: z.string().optional(),
+  FLY_IO_API_URL: z.string().url().optional(),
+  FLY_IO_ORG_SLUG: z.string().optional(),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/hooks/useJobs.tsx

@@ -8,6 +8,10 @@ export type ProjectJob = UseDataFunctionReturn<typeof loader>["projectJobs"][num
 
 export const jobsMatchId =
   "routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam";
+
+// This is only used in the JobsMenu component, which is the breadcrumb job list dropdown.
+// This dropdown is only shown once you have selected a job, so we can assume that
+// the route above has loaded and we can use the data from it.
 export function useOptionalJobs(matches?: RouteMatch[]) {
   const routeMatch = useTypedMatchesData<typeof loader>({
     id: jobsMatchId,

apps/webapp/app/models/backgroundTaskSecret.server.ts

@@ -0,0 +1,136 @@
+import {
+  BackgroundTaskSecret,
+  BackgroundTaskVersion,
+  SecretReference,
+} from "@trigger.dev/database";
+import { z } from "zod";
+import { $transaction, PrismaClientOrTransaction } from "~/db.server";
+import { SecretStore, getSecretStore } from "~/services/secrets/secretStore.server";
+
+const BackgroundTaskSecretSchema = z.object({
+  secret: z.string(),
+});
+
+export async function createBackgroundTaskSecret(
+  prisma: PrismaClientOrTransaction,
+  version: BackgroundTaskVersion,
+  key: string,
+  value: string
+) {
+  const secretKey = `${version.environmentId}:${key}`;
+
+  return await $transaction(prisma, async (tx) => {
+    const newSecret = await tx.backgroundTaskSecret.create({
+      data: {
+        key,
+        backgroundTaskVersion: {
+          connect: {
+            id: version.id,
+          },
+        },
+        secretReference: {
+          connectOrCreate: {
+            where: {
+              key: secretKey,
+            },
+            create: {
+              key: secretKey,
+              provider: "DATABASE",
+            },
+          },
+        },
+      },
+      include: {
+        secretReference: true,
+      },
+    });
+
+    const secretStoreProvider = getSecretStore("DATABASE", { prismaClient: tx });
+    const secretStore = new SecretStore(secretStoreProvider);
+
+    await secretStore.setSecret(secretKey, { secret: value });
+
+    return newSecret;
+  });
+}
+
+export async function updateBackgroundTaskSecret(
+  prisma: PrismaClientOrTransaction,
+  version: BackgroundTaskVersion,
+  key: string,
+  value: string
+) {
+  const secretKey = `${version.environmentId}:${key}`;
+
+  return await $transaction(prisma, async (tx) => {
+    const updatedSecret = await tx.backgroundTaskSecret.upsert({
+      where: {
+        backgroundTaskVersionId_key: {
+          backgroundTaskVersionId: version.id,
+          key,
+        },
+      },
+      create: {
+        key,
+        backgroundTaskVersion: {
+          connect: {
+            id: version.id,
+          },
+        },
+        secretReference: {
+          connectOrCreate: {
+            where: {
+              key: secretKey,
+            },
+            create: {
+              key: secretKey,
+              provider: "DATABASE",
+            },
+          },
+        },
+      },
+      update: {},
+      include: {
+        secretReference: true,
+      },
+    });
+
+    const secretStoreProvider = getSecretStore("DATABASE", { prismaClient: tx });
+    const secretStore = new SecretStore(secretStoreProvider);
+
+    await secretStore.setSecret(secretKey, { secret: value });
+
+    return updatedSecret;
+  });
+}
+
+export async function deleteBackgroundTaskSecret(prisma: PrismaClientOrTransaction, id: string) {
+  return await $transaction(prisma, async (tx) => {
+    const secret = await tx.backgroundTaskSecret.delete({
+      where: {
+        id,
+      },
+      include: {
+        secretReference: true,
+      },
+    });
+
+    if (!secret) {
+      return;
+    }
+
+    const secretStoreProvider = getSecretStore("DATABASE", { prismaClient: tx });
+    const secretStore = new SecretStore(secretStoreProvider);
+
+    await secretStore.deleteSecret(secret.secretReference.key);
+  });
+}
+
+export async function resolveBackgroundTaskSecret(reference: SecretReference) {
+  const secretStoreProvider = getSecretStore(reference.provider);
+  const secretStore = new SecretStore(secretStoreProvider);
+
+  const secretRecord = await secretStore.getSecret(BackgroundTaskSecretSchema, reference.key);
+
+  return secretRecord?.secret;
+}

apps/webapp/app/models/jobRunExecution.server.ts

@@ -0,0 +1,62 @@
+import { JobRun, JobRunExecution } from "@trigger.dev/database";
+import { PrismaClientOrTransaction } from "~/db.server";
+import { executionWorker } from "~/services/worker.server";
+
+export async function enqueueRunExecutionV1(
+  execution: JobRunExecution,
+  queueId: string,
+  concurrency: number,
+  tx: PrismaClientOrTransaction,
+  runAt?: Date
+) {
+  const job = await executionWorker.enqueue(
+    "performRunExecution",
+    {
+      id: execution.id,
+    },
+    {
+      queueName: `job:queue:${queueId}`,
+      tx,
+      runAt,
+      jobKey: `execution:${execution.runId}`,
+    }
+  );
+}
+
+export type EnqueueRunExecutionV2Options = {
+  runAt?: Date;
+  resumeTaskId?: string;
+  isRetry?: boolean;
+  skipRetrying?: boolean;
+  executionCount?: number;
+};
+
+export async function enqueueRunExecutionV2(
+  run: JobRun,
+  tx: PrismaClientOrTransaction,
+  options: EnqueueRunExecutionV2Options = {}
+) {
+  const job = await executionWorker.enqueue(
+    "performRunExecutionV2",
+    {
+      id: run.id,
+      reason: run.status === "PREPROCESSING" ? "PREPROCESS" : "EXECUTE_JOB",
+      resumeTaskId: options.resumeTaskId,
+      isRetry: typeof options.isRetry === "boolean" ? options.isRetry : false,
+    },
+    {
+      tx,
+      runAt: options.runAt,
+      jobKey: `job_run:${run.id}:${options.executionCount ?? 0}${
+        options.resumeTaskId ? `:task:${options.resumeTaskId}` : ""
+      }`,
+      maxAttempts: options.skipRetrying ? 1 : undefined,
+    }
+  );
+}
+
+export async function dequeueRunExecutionV2(run: JobRun, tx: PrismaClientOrTransaction) {
+  return await executionWorker.dequeue(`job_run:${run.id}`, {
+    tx,
+  });
+}

apps/webapp/app/models/message.server.ts

@@ -102,6 +102,25 @@ export async function jsonWithSuccessMessage(
   });
 }
 
+export async function jsonWithErrorMessage(
+  data: any,
+  request: Request,
+  message: string,
+  options?: ToastMessageOptions
+) {
+  const session = await getSession(request.headers.get("cookie"));
+
+  setErrorMessage(session, message, options);
+
+  return json(data, {
+    headers: {
+      "Set-Cookie": await commitSession(session, {
+        expires: new Date(Date.now() + ONE_YEAR),
+      }),
+    },
+  });
+}
+
 export async function redirectWithSuccessMessage(
   path: string,
   request: Request,

apps/webapp/app/models/organization.server.ts

@@ -49,6 +49,7 @@ export function getOrganizations({ userId }: { userId: User["id"] }) {
               jobs: {
                 where: {
                   internal: false,
+                  deletedAt: null,
                 },
               },
             },

apps/webapp/app/models/sourceConnection.server.ts

@@ -0,0 +1,61 @@
+import { ExternalAccount, Integration, TriggerSource } from "@trigger.dev/database";
+import { ConnectionAuth } from "@trigger.dev/core";
+import { PrismaClientOrTransaction } from "~/db.server";
+import { integrationAuthRepository } from "~/services/externalApis/integrationAuthRepository.server";
+import { logger } from "~/services/logger.server";
+
+type ResolvableTriggerSource = TriggerSource & {
+  integration: Integration;
+  externalAccount: ExternalAccount | null;
+};
+
+export async function resolveSourceConnection(
+  tx: PrismaClientOrTransaction,
+  source: ResolvableTriggerSource
+): Promise<ConnectionAuth | undefined> {
+  if (source.integration.authSource !== "HOSTED") return;
+
+  const connection = await getConnection(tx, source);
+
+  if (!connection) {
+    logger.error(
+      `Integration connection not found for source ${source.id}, integration ${source.integration.id}`
+    );
+    return;
+  }
+
+  const response = await integrationAuthRepository.getCredentials(connection);
+
+  if (!response) {
+    return;
+  }
+
+  return {
+    type: "oauth2",
+    scopes: response.scopes,
+    accessToken: response.accessToken,
+  };
+}
+
+function getConnection(tx: PrismaClientOrTransaction, source: ResolvableTriggerSource) {
+  if (source.externalAccount) {
+    return tx.integrationConnection.findFirst({
+      where: {
+        integrationId: source.integration.id,
+        externalAccountId: source.externalAccount.id,
+      },
+      include: {
+        dataReference: true,
+      },
+    });
+  }
+
+  return tx.integrationConnection.findFirst({
+    where: {
+      integrationId: source.integration.id,
+    },
+    include: {
+      dataReference: true,
+    },
+  });
+}

apps/webapp/app/models/task.server.ts

@@ -1,5 +1,6 @@
 import type { Task, TaskAttempt } from "@trigger.dev/database";
 import { ServerTask } from "@trigger.dev/core";
+import { PrismaClientOrTransaction } from "~/db.server";
 
 export type TaskWithAttempts = Task & { attempts: TaskAttempt[] };
 
@@ -25,3 +26,20 @@ export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask
     operation: task.operation,
   };
 }
+
+export type KitchenSinkTask = NonNullable<Awaited<ReturnType<typeof findKitchenSinkTask>>>;
+
+export async function findKitchenSinkTask(prisma: PrismaClientOrTransaction, id: string) {
+  return prisma.task.findUnique({
+    where: { id },
+    include: {
+      attempts: true,
+      run: {
+        include: {
+          environment: true,
+          queue: true,
+        },
+      },
+    },
+  });
+}

apps/webapp/app/platform/zodWorker.server.ts

@@ -51,6 +51,7 @@ const AddJobResultsSchema = z.array(GraphileJobSchema);
 export type ZodTasks<TConsumerSchema extends MessageCatalogSchema> = {
   [K in keyof TConsumerSchema]: {
     queueName?: string | ((payload: z.infer<TConsumerSchema[K]>) => string);
+    jobKey?: string | ((payload: z.infer<TConsumerSchema[K]>) => string | undefined);
     priority?: number;
     maxAttempts?: number;
     jobKeyMode?: "replace" | "preserve_run_at" | "unsafe_dedupe";
@@ -76,7 +77,12 @@ export type ZodWorkerEnqueueOptions = TaskSpec & {
   tx?: PrismaClientOrTransaction;
 };
 
+export type ZodWorkerDequeueOptions = {
+  tx?: PrismaClientOrTransaction;
+};
+
 export type ZodWorkerOptions<TMessageCatalog extends MessageCatalogSchema> = {
+  name: string;
   runnerOptions: RunnerOptions;
   prisma: PrismaClient;
   schema: TMessageCatalog;
@@ -85,6 +91,7 @@ export type ZodWorkerOptions<TMessageCatalog extends MessageCatalogSchema> = {
 };
 
 export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
+  #name: string;
   #schema: TMessageCatalog;
   #prisma: PrismaClient;
   #runnerOptions: RunnerOptions;
@@ -93,6 +100,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
   #runner?: GraphileRunner;
 
   constructor(options: ZodWorkerOptions<TMessageCatalog>) {
+    this.#name = options.name;
     this.#schema = options.schema;
     this.#prisma = options.prisma;
     this.#runnerOptions = options.runnerOptions;
@@ -100,12 +108,16 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     this.#recurringTasks = options.recurringTasks;
   }
 
+  get graphileWorkerSchema() {
+    return this.#runnerOptions.schema ?? "graphile_worker";
+  }
+
   public async initialize(): Promise<boolean> {
     if (this.#runner) {
       return true;
     }
 
-    logger.debug("Initializing worker queue with options", {
+    this.#logDebug("Initializing worker queue with options", {
       runnerOptions: this.#runnerOptions,
     });
 
@@ -121,6 +133,54 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       throw new Error("Failed to initialize worker queue");
     }
 
+    this.#runner?.events.on("pool:create", ({ workerPool }) => {
+      this.#logDebug("pool:create");
+    });
+
+    this.#runner?.events.on("pool:listen:connecting", ({ workerPool, attempts }) => {
+      this.#logDebug("pool:create", { attempts });
+    });
+
+    this.#runner?.events.on("pool:listen:success", ({ workerPool, client }) => {
+      this.#logDebug("pool:listen:success");
+    });
+
+    this.#runner?.events.on("pool:listen:error", ({ error }) => {
+      this.#logDebug("pool:listen:error", { error });
+    });
+
+    this.#runner?.events.on("pool:gracefulShutdown", ({ message }) => {
+      this.#logDebug("pool:gracefulShutdown", { workerMessage: message });
+    });
+
+    this.#runner?.events.on("pool:gracefulShutdown:error", ({ error }) => {
+      this.#logDebug("pool:gracefulShutdown:error", { error });
+    });
+
+    this.#runner?.events.on("worker:create", ({ worker }) => {
+      this.#logDebug("worker:create", { workerId: worker.workerId });
+    });
+
+    this.#runner?.events.on("worker:release", ({ worker }) => {
+      this.#logDebug("worker:release", { workerId: worker.workerId });
+    });
+
+    this.#runner?.events.on("worker:stop", ({ worker, error }) => {
+      this.#logDebug("worker:stop", { workerId: worker.workerId, error });
+    });
+
+    this.#runner?.events.on("worker:fatalError", ({ worker, error, jobError }) => {
+      this.#logDebug("worker:fatalError", { workerId: worker.workerId, error, jobError });
+    });
+
+    this.#runner?.events.on("gracefulShutdown", ({ signal }) => {
+      this.#logDebug("gracefulShutdown", { signal });
+    });
+
+    this.#runner?.events.on("stop", () => {
+      this.#logDebug("stop");
+    });
+
     return true;
   }
 
@@ -133,23 +193,35 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     payload: z.infer<TMessageCatalog[K]>,
     options?: ZodWorkerEnqueueOptions
   ): Promise<GraphileJob> {
-    if (!this.#runner) {
-      throw new Error("Worker not initialized");
-    }
-
     const task = this.#tasks[identifier];
 
-    const optionsWithoutTx = omit(options ?? {}, ["tx"]);
+    const optionsWithoutTx = removeUndefinedKeys(omit(options ?? {}, ["tx"]));
+    const taskWithoutJobKey = omit(task, ["jobKey"]);
 
+    // Make sure options passed in to enqueue take precedence over task options
     const spec = {
+      ...taskWithoutJobKey,
       ...optionsWithoutTx,
-      ...task,
     };
 
     if (typeof task.queueName === "function") {
       spec.queueName = task.queueName(payload);
     }
 
+    if (typeof task.jobKey === "function") {
+      const jobKey = task.jobKey(payload);
+
+      if (jobKey) {
+        spec.jobKey = jobKey;
+      }
+    }
+
+    logger.debug("Enqueuing worker task", {
+      identifier,
+      payload,
+      spec,
+    });
+
     const job = await this.#addJob(
       identifier as string,
       payload,
@@ -167,6 +239,17 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     return job;
   }
 
+  public async dequeue(
+    jobKey: string,
+    option?: ZodWorkerDequeueOptions
+  ): Promise<GraphileJob | undefined> {
+    const results = await this.#removeJob(jobKey, option?.tx ?? this.#prisma);
+
+    logger.debug("dequeued worker task", { results, jobKey });
+
+    return results;
+  }
+
   async #addJob(
     identifier: string,
     payload: unknown,
@@ -174,7 +257,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     tx: PrismaClientOrTransaction
   ) {
     const results = await tx.$queryRawUnsafe(
-      `SELECT * FROM graphile_worker.add_job(
+      `SELECT * FROM ${this.graphileWorkerSchema}.add_job(
           identifier => $1::text,
           payload => $2::json,
           queue_name => $3::text,
@@ -192,8 +275,8 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       spec.maxAttempts || null,
       spec.jobKey || null,
       spec.priority || null,
-      spec.jobKeyMode || null,
-      spec.flags || null
+      spec.flags || null,
+      spec.jobKeyMode || null
     );
 
     const rows = AddJobResultsSchema.safeParse(results);
@@ -209,6 +292,32 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     return job as GraphileJob;
   }
 
+  async #removeJob(jobKey: string, tx: PrismaClientOrTransaction) {
+    try {
+      const result = await tx.$queryRawUnsafe(
+        `SELECT * FROM ${this.graphileWorkerSchema}.remove_job(
+          job_key => $1::text
+        )`,
+        jobKey
+      );
+      const job = AddJobResultsSchema.safeParse(result);
+
+      if (!job.success) {
+        logger.debug("results returned from remove_job could not be parsed", {
+          error: job.error.flatten(),
+          result,
+          jobKey,
+        });
+
+        return;
+      }
+
+      return job.data[0] as GraphileJob;
+    } catch (e) {
+      throw new Error(`Failed to remove job from queue, ${e}}`);
+    }
+  }
+
   #createTaskListFromTasks() {
     const taskList: TaskList = {};
 
@@ -324,4 +433,17 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       throw error;
     }
   }
+
+  #logDebug(message: string, args?: any) {
+    logger.debug(`[worker][${this.#name}] ${message}`, args);
+  }
+}
+
+function removeUndefinedKeys<T extends object>(obj: T): T {
+  for (let key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key) && obj[key] === undefined) {
+      delete obj[key];
+    }
+  }
+  return obj;
 }

apps/webapp/app/presenters/IntegrationClientPresenter.server.ts

@@ -63,6 +63,7 @@ export class IntegrationClientPresenter {
                     slug: projectSlug,
                   },
                   internal: false,
+                  deletedAt: null,
                 },
               },
             },

apps/webapp/app/presenters/IntegrationsPresenter.server.ts

@@ -71,6 +71,7 @@ export class IntegrationsPresenter {
                     slug: projectSlug,
                   },
                   internal: false,
+                  deletedAt: null,
                 },
               },
             },

apps/webapp/app/presenters/JobListPresenter.server.ts

@@ -47,6 +47,7 @@ export class JobListPresenter {
                 version: true,
                 eventSpecification: true,
                 properties: true,
+                status: true,
                 runs: {
                   select: {
                     createdAt: true,
@@ -92,6 +93,7 @@ export class JobListPresenter {
       },
       where: {
         internal: false,
+        deletedAt: null,
         organization: orgWhere,
         project: {
           slug: projectSlug,
@@ -162,11 +164,19 @@ export class JobListPresenter {
           properties = [...properties, ...versionProperties];
         }
 
+        const environments = job.aliases.map((alias) => ({
+          type: alias.environment.type,
+          enabled: alias.version.status === "ACTIVE",
+          lastRun: alias.version.runs.at(0)?.createdAt,
+          version: alias.version.version,
+        }));
+
         return {
           id: job.id,
           slug: job.slug,
           title: job.title,
           version: alias.version.version,
+          status: alias.version.status,
           dynamic: job.dynamicTriggers.length > 0,
           event: {
             title: eventSpecification.title,
@@ -179,6 +189,7 @@ export class JobListPresenter {
           ),
           lastRun,
           properties,
+          environments,
         };
       })
       .filter(Boolean);

apps/webapp/app/presenters/ProjectPresenter.server.ts

@@ -80,6 +80,7 @@ export class ProjectPresenter {
           },
           where: {
             internal: false,
+            deletedAt: null,
           },
           orderBy: [{ title: "asc" }],
         },

apps/webapp/app/presenters/TaskDetailsPresenter.server.ts

@@ -1,4 +1,4 @@
-import { DisplayPropertiesSchema, StyleSchema } from "@trigger.dev/core";
+import { StyleSchema } from "@trigger.dev/core";
 import { PrismaClient, prisma } from "~/db.server";
 import { mergeProperties } from "~/utils/mergeProperties.server";
 
@@ -7,8 +7,6 @@ type DetailsProps = {
   userId: string;
 };
 
-export type DetailedTask = NonNullable<Awaited<ReturnType<TaskDetailsPresenter["call"]>>>;
-
 export class TaskDetailsPresenter {
   #prismaClient: PrismaClient;
 
@@ -87,6 +85,7 @@ export class TaskDetailsPresenter {
 
     return {
       ...task,
+      output: task.output ? JSON.stringify(task.output, null, 2) : undefined,
       connection: task.runConnection,
       params: task.params as Record<string, any>,
       properties: mergeProperties(task.properties, task.outputProperties),

apps/webapp/app/presenters/TriggerSourcePresenter.server.ts

@@ -62,6 +62,22 @@ export class TriggerSourcePresenter {
             },
           },
         },
+        dynamicTrigger: {
+          select: {
+            id: true,
+            slug: true,
+            sourceRegistrationJob: {
+              select: {
+                job: {
+                  select: {
+                    id: true,
+                    slug: true,
+                  },
+                },
+              },
+            },
+          },
+        },
       },
       where: {
         id: triggerSourceId,
@@ -73,10 +89,15 @@ export class TriggerSourcePresenter {
     }
 
     const runListPresenter = new RunListPresenter(this.#prismaClient);
-    const runList = trigger.sourceRegistrationJob
+    const jobSlug = getJobSlug(
+      trigger.sourceRegistrationJob?.job.slug,
+      trigger.dynamicTrigger?.sourceRegistrationJob?.job.slug
+    );
+
+    const runList = jobSlug
       ? await runListPresenter.call({
           userId,
-          jobSlug: trigger.sourceRegistrationJob.job.slug,
+          jobSlug,
           organizationSlug,
           projectSlug,
           direction,
@@ -95,7 +116,21 @@ export class TriggerSourcePresenter {
         params: trigger.params,
         registrationJob: trigger.sourceRegistrationJob?.job,
         runList,
+        dynamic: trigger.dynamicTrigger
+          ? { id: trigger.dynamicTrigger.id, slug: trigger.dynamicTrigger.slug }
+          : undefined,
       },
     };
   }
 }
+
+function getJobSlug(
+  sourceRegistrationJobSlug: string | undefined,
+  dynamicSourceRegistrationJobSlug: string | undefined
+) {
+  if (sourceRegistrationJobSlug) {
+    return sourceRegistrationJobSlug;
+  }
+
+  return dynamicSourceRegistrationJobSlug;
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam._index/route.tsx

@@ -73,8 +73,6 @@ export default function Page() {
 
   const { filterText, setFilterText, filteredItems } = useFilterJobs(jobs);
 
-  const { width, height } = useWindowSize();
-
   return (
     <PageContainer>
       <PageHeader>
@@ -104,7 +102,6 @@ export default function Page() {
                       </Callout>
                     )}
                     <div className="mb-2 flex flex-col">
-                      <Header2 spacing>Jobs</Header2>
                       <div className="flex w-full">
                         <Input
                           placeholder="Search Jobs"
@@ -122,9 +119,11 @@ export default function Page() {
                       noResultsText={`No Jobs match ${filterText}. Try a different search
           query.`}
                     />
-                    {jobs.length === 1 && jobs.every((r) => r.lastRun === undefined) && (
-                      <RunYourJobPrompt />
-                    )}
+                    {jobs.length === 1 &&
+                      jobs.every((r) => r.lastRun === undefined) &&
+                      jobs.every((i) => i.hasIntegrationsRequiringAction === false) && (
+                        <RunYourJobPrompt />
+                      )}
                   </>
                 ) : (
                   <HowToSetupYourProject />
@@ -194,7 +193,7 @@ function ExampleJobs() {
           <a
             href={example.codeLink}
             key={example.title}
-            className="flex w-full items-center rounded border-b border-slate-800 py-2 transition hover:border-transparent hover:bg-slate-800"
+            className="flex w-full items-center rounded border-b border-uiBorder py-2 transition hover:border-transparent hover:bg-slate-800"
           >
             {example.icon}
             <Paragraph variant="small">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.integrations/route.tsx

@@ -355,7 +355,7 @@ function ConnectedIntegrationsList({
                         <TableCell to={path}>
                           <DateTime date={client.createdAt} includeSeconds={false} />
                         </TableCell>
-                        <TableCellChevron to={path} />
+                        <TableCellChevron to={path} isSticky />
                       </TableRow>
                     );
                   })}
@@ -454,7 +454,7 @@ function IntegrationsWithMissingFields({
                     integration={integration}
                     organizationId={organizationId}
                     button={
-                      <ChevronRightIcon className="h-4 w-4 text-slate-700 transition group-hover:text-bright" />
+                      <ChevronRightIcon className="h-4 w-4 text-dimmed transition group-hover:text-bright" />
                     }
                     callbackUrl={callbackUrl}
                     existingIntegration={client}
@@ -482,7 +482,7 @@ function AddIntegrationConnection({
   icon?: string;
 }) {
   return (
-    <div className="group flex h-11 w-full items-center gap-2 rounded-md p-1 pr-3 transition hover:bg-slate-850">
+    <div className="group flex h-11 w-full items-center gap-2 rounded-md p-1 pr-3 transition hover:bg-slate-900">
       <NamedIconInBox
         name={icon ?? identifier}
         className="h-9 w-9 flex-none transition group-hover:border-slate-750"

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route.tsx

@@ -1,5 +1,7 @@
-import { LoaderArgs } from "@remix-run/server-runtime";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { Await, useLoaderData } from "@remix-run/react";
+import { LoaderArgs, SerializeFrom, defer } from "@remix-run/server-runtime";
+import { Suspense } from "react";
+import { Spinner } from "~/components/primitives/Spinner";
 import { TaskDetail } from "~/components/run/TaskDetail";
 import { TaskDetailsPresenter } from "~/presenters/TaskDetailsPresenter.server";
 import { requireUserId } from "~/services/session.server";
@@ -10,23 +12,26 @@ export const loader = async ({ request, params }: LoaderArgs) => {
   const { taskParam } = TaskParamsSchema.parse(params);
 
   const presenter = new TaskDetailsPresenter();
-  const task = await presenter.call({
+  const taskPromise = presenter.call({
     userId,
     id: taskParam,
   });
 
-  if (!task) {
-    throw new Response(null, {
-      status: 404,
-    });
-  }
-
-  return typedjson({
-    task,
+  return defer({
+    taskPromise,
   });
 };
 
+export type DetailedTask = NonNullable<Awaited<SerializeFrom<typeof loader>["taskPromise"]>>;
+
 export default function Page() {
-  const { task } = useTypedLoaderData<typeof loader>();
-  return <TaskDetail task={task} />;
+  const { taskPromise } = useLoaderData<typeof loader>();
+
+  return (
+    <Suspense fallback={<Spinner />}>
+      <Await resolve={taskPromise} errorElement={<p>Error loading task!</p>}>
+        {(resolvedTask) => resolvedTask && <TaskDetail task={resolvedTask as any} />}
+      </Await>
+    </Suspense>
+  );
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.settings/route.tsx

@@ -1,11 +1,68 @@
-import { ComingSoon } from "~/components/ComingSoon";
+import { JobEnvironment, JobStatusTable } from "~/components/JobsStatusTable";
+import { HowToDisableAJob } from "~/components/helpContent/HelpContentText";
+import { DeleteJobDialogContent } from "~/components/jobs/DeleteJobModalContent";
+import { Button } from "~/components/primitives/Buttons";
+import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog";
+import { Header2 } from "~/components/primitives/Headers";
+import { Help, HelpContent, HelpTrigger } from "~/components/primitives/Help";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { useJob } from "~/hooks/useJob";
+import { useOrganization } from "~/hooks/useOrganizations";
+import { useProject } from "~/hooks/useProject";
+import { cn } from "~/utils/cn";
+import { projectJobsPath, projectPath } from "~/utils/pathBuilder";
 
 export default function Page() {
+  const job = useJob();
+  const organization = useOrganization();
+  const project = useProject();
+
   return (
-    <ComingSoon
-      title="Job settings"
-      description="Disable a Job, archive it and more."
-      icon="settings"
-    />
+    <Help defaultOpen>
+      {(open) => (
+        <div className={cn("grid h-fit gap-4", open ? "grid-cols-2" : "grid-cols-1")}>
+          <div className="w-full">
+            <div className="flex items-center justify-between">
+              <Header2 className="mb-2 flex items-center gap-1">Environments</Header2>
+              <HelpTrigger title="How do disable a Job?" />
+            </div>
+            <JobStatusTable environments={job.environments} />
+            <div className="mt-4 flex w-full items-center justify-end gap-x-3">
+              {job.status === "ACTIVE" && (
+                <Paragraph variant="small">
+                  Disable this Job in all environments before deleting
+                </Paragraph>
+              )}
+
+              <Dialog>
+                <DialogTrigger asChild>
+                  <Button
+                    variant="danger/small"
+                    leadingIconClassName="text-bright"
+                    LeadingIcon="trash-can"
+                  >
+                    I want to delete this Job
+                  </Button>
+                </DialogTrigger>
+                <DialogContent>
+                  <DialogHeader>
+                    <DeleteJobDialogContent
+                      title={job.title}
+                      slug={job.slug}
+                      environments={job.environments}
+                      id={job.id}
+                      redirectTo={projectJobsPath(organization, project)}
+                    />
+                  </DialogHeader>
+                </DialogContent>
+              </Dialog>
+            </div>
+          </div>
+          <HelpContent title="How to disable a Job">
+            <HowToDisableAJob id={job.slug} version={job.version} name={job.title} />
+          </HelpContent>
+        </div>
+      )}
+    </Help>
   );
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam/route.tsx

@@ -2,6 +2,7 @@ import { Outlet, useLocation } from "@remix-run/react";
 import type { LoaderArgs } from "@remix-run/server-runtime";
 import { Fragment } from "react";
 import { typedjson } from "remix-typedjson";
+import { JobStatusBadge } from "~/components/jobs/JobStatusBadge";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { JobsMenu } from "~/components/navigation/JobsMenu";
 import { BreadcrumbLink } from "~/components/navigation/NavBar";
@@ -134,6 +135,18 @@ export default function Job() {
                 }
               />
             )}
+            <PageInfoProperty
+              icon="pulse"
+              label={"STATUS"}
+              value={
+                <JobStatusBadge
+                  enabled={job.status === "ACTIVE"}
+                  hasIntegrationsRequiringAction={job.hasIntegrationsRequiringAction}
+                  hasRuns={job.lastRun !== undefined}
+                  badgeSize="small"
+                />
+              }
+            />
           </PageInfoGroup>
           <PageInfoGroup alignment="right">
             <Paragraph variant="extra-small" className="text-slate-600">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam/route.tsx

@@ -43,6 +43,7 @@ import { parse } from "@conform-to/zod";
 import { z } from "zod";
 import { ActivateSourceService } from "~/services/sources/activateSource.server";
 import { redirectWithSuccessMessage } from "~/models/message.server";
+import { nanoid } from "nanoid";
 
 export const loader = async ({ request, params }: LoaderArgs) => {
   const user = await requireUser(request);
@@ -90,7 +91,7 @@ export const action: ActionFunction = async ({ request, params }) => {
   try {
     const service = new ActivateSourceService();
 
-    const result = await service.call(triggerParam, submission.value.jobId);
+    const result = await service.call(triggerParam);
 
     return redirectWithSuccessMessage(
       externalTriggerPath({ slug: organizationSlug }, { slug: projectParam }, { id: triggerParam }),
@@ -167,6 +168,17 @@ export default function Page() {
                 <NamedIcon name={trigger.active ? "active" : "inactive"} className="h-4 w-4" />
               }
             />
+            {trigger.dynamic && (
+              <PageInfoProperty
+                label="Dynamic"
+                value={
+                  <span className="flex items-center gap-0.5">
+                    <NamedIcon name="dynamic" className="h-4 w-4" />
+                    {trigger.dynamic.slug}
+                  </span>
+                }
+              />
+            )}
             <PageInfoProperty
               label="Environment"
               value={<EnvironmentLabel environment={trigger.environment} />}
@@ -206,7 +218,7 @@ export default function Page() {
                   </Button>
                 </Callout>
               </Form>
-            ) : (
+            ) : trigger.dynamic ? null : (
               <Callout variant="error" className="justiy-between mb-4 items-center">
                 This External Trigger hasn't registered successfully. Contact support for help:{" "}
                 {trigger.id}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam_.runs.$runParam.tasks.$taskParam/route.tsx

@@ -1,5 +1,7 @@
-import { LoaderArgs } from "@remix-run/server-runtime";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { Await, useLoaderData } from "@remix-run/react";
+import { LoaderArgs, defer } from "@remix-run/server-runtime";
+import { Suspense } from "react";
+import { Spinner } from "~/components/primitives/Spinner";
 import { TaskDetail } from "~/components/run/TaskDetail";
 import { TaskDetailsPresenter } from "~/presenters/TaskDetailsPresenter.server";
 import { requireUserId } from "~/services/session.server";
@@ -10,23 +12,24 @@ export const loader = async ({ request, params }: LoaderArgs) => {
   const { taskParam } = TriggerSourceRunTaskParamsSchema.parse(params);
 
   const presenter = new TaskDetailsPresenter();
-  const task = await presenter.call({
+  const taskPromise = presenter.call({
     userId,
     id: taskParam,
   });
 
-  if (!task) {
-    throw new Response(null, {
-      status: 404,
-    });
-  }
-
-  return typedjson({
-    task,
+  return defer({
+    taskPromise,
   });
 };
 
 export default function Page() {
-  const { task } = useTypedLoaderData<typeof loader>();
-  return <TaskDetail task={task} />;
+  const { taskPromise } = useLoaderData<typeof loader>();
+
+  return (
+    <Suspense fallback={<Spinner />}>
+      <Await resolve={taskPromise} errorElement={<p>Error loading task!</p>}>
+        {(resolvedTask) => resolvedTask && <TaskDetail task={resolvedTask as any} />}
+      </Await>
+    </Suspense>
+  );
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.team/route.tsx

@@ -103,7 +103,7 @@ export default function Page() {
       <OrgAdminHeader />
       <PageBody>
         <Header2>Members</Header2>
-        <ul className="flex w-full max-w-md flex-col divide-y divide-slate-800 border-b border-slate-800">
+        <ul className="flex w-full max-w-md flex-col divide-y divide-uiBorder border-b border-uiBorder">
           {members.map((member) => (
             <li key={member.user.id} className="flex items-center gap-x-4 py-4">
               <UserAvatar

apps/webapp/app/routes/_docker.v2.$.ts

@@ -0,0 +1,10 @@
+import { ActionArgs, LoaderArgs } from "@remix-run/server-runtime";
+import { proxyToRegistry } from "~/services/docker/registryProxy.server";
+
+export async function action({ request }: ActionArgs) {
+  return await proxyToRegistry(request);
+}
+
+export async function loader({ request, params }: LoaderArgs) {
+  return await proxyToRegistry(request);
+}

apps/webapp/app/routes/_docker.v2.ts

@@ -0,0 +1,6 @@
+import { LoaderArgs } from "@remix-run/server-runtime";
+import { proxyToRegistry } from "~/services/docker/registryProxy.server";
+
+export async function loader({ request }: LoaderArgs) {
+  return await proxyToRegistry(request);
+}

apps/webapp/app/routes/api.internal.stripe_webhooks.ts

@@ -19,7 +19,10 @@ export async function action({ request }: ActionArgs) {
     },
     body: JSON.stringify({
       title: body.type,
-      content: body,
+      content: {
+        ...body,
+        example: { id: body.type, name: body.type, icon: "stripe", payload: body.data.object },
+      },
       readOnly: true,
     }),
   });

apps/webapp/app/routes/api.v1.$endpointSlug.sources.$id.ts

@@ -1,10 +1,10 @@
 import type { ActionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import { UpdateTriggerSourceBodySchema } from "@trigger.dev/core";
+import { UpdateTriggerSourceBodyV1Schema } from "@trigger.dev/core";
 import { z } from "zod";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { UpdateSourceService } from "~/services/sources/updateSource.server";
+import { UpdateSourceServiceV1 } from "~/services/sources/updateSourceV1.server";
 
 const ParamsSchema = z.object({
   endpointSlug: z.string(),
@@ -40,13 +40,13 @@ export async function action({ request, params }: ActionArgs) {
   // Now parse the request body
   const anyBody = await request.json();
 
-  const body = UpdateTriggerSourceBodySchema.safeParse(anyBody);
+  const body = UpdateTriggerSourceBodyV1Schema.safeParse(anyBody);
 
   if (!body.success) {
     return json({ error: "Invalid request body" }, { status: 400 });
   }
 
-  const service = new UpdateSourceService();
+  const service = new UpdateSourceServiceV1();
 
   try {
     const source = await service.call({

apps/webapp/app/routes/api.v1.$endpointSlug.triggers.$id.registrations.$key.ts

@@ -1,10 +1,10 @@
 import type { ActionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import { RegisterTriggerBodySchema } from "@trigger.dev/core";
+import { RegisterTriggerBodySchemaV1 } from "@trigger.dev/core";
 import { z } from "zod";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { RegisterTriggerSourceService } from "~/services/triggers/registerTriggerSource.server";
+import { RegisterTriggerSourceServiceV1 } from "~/services/triggers/registerTriggerSourceV1.server";
 
 const ParamsSchema = z.object({
   endpointSlug: z.string(),
@@ -41,13 +41,13 @@ export async function action({ request, params }: ActionArgs) {
   // Now parse the request body
   const anyBody = await request.json();
 
-  const body = RegisterTriggerBodySchema.safeParse(anyBody);
+  const body = RegisterTriggerBodySchemaV1.safeParse(anyBody);
 
   if (!body.success) {
     return json({ error: "Invalid request body" }, { status: 400 });
   }
 
-  const service = new RegisterTriggerSourceService();
+  const service = new RegisterTriggerSourceServiceV1();
 
   try {
     const registration = await service.call({

apps/webapp/app/routes/api.v1.background.artifacts.$id.images.ts

@@ -0,0 +1,66 @@
+import type { ActionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { CreateBackgroundTaskImageRequestBodySchema } from "@trigger.dev/core";
+import { z } from "zod";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CreateBackgroundTaskImageService } from "~/services/backgroundTasks/createBackgroundTaskImage.server";
+
+const ParamsSchema = z.object({
+  id: z.string(),
+});
+
+export async function action({ request, params }: ActionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid request params" }, { status: 400 });
+  }
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  const body = CreateBackgroundTaskImageRequestBodySchema.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new CreateBackgroundTaskImageService();
+
+  try {
+    const image = await service.call(
+      authenticationResult.environment,
+      parsedParams.data.id,
+      body.data
+    );
+
+    if (!image) {
+      return json(
+        {
+          error: `Unable to create background task image`,
+        },
+        { status: 500 }
+      );
+    }
+
+    return json(image);
+  } catch (error) {
+    if (error instanceof Error) {
+      return json({ error: error.message }, { status: 400 });
+    }
+
+    return json({ error: "Something went wrong" }, { status: 500 });
+  }
+}

apps/webapp/app/routes/api.v1.background.artifacts.$id[.tgz].ts

@@ -0,0 +1,142 @@
+import { Response } from "@remix-run/node";
+import { LoaderArgs, json } from "@remix-run/server-runtime";
+import { BackgroundTask, BackgroundTaskArtifact, PrismaClient } from "@trigger.dev/database";
+import archiver from "archiver";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+
+const ParamsSchema = z.object({
+  id: z.string(),
+});
+
+export async function loader({ params }: LoaderArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid request params" }, { status: 400 });
+  }
+
+  const service = new GenerateBackgroundTaskArtifactArchiveService();
+
+  const results = await service.call(parsedParams.data.id);
+
+  if (!results) {
+    return json({ error: "Something went wrong" }, { status: 500 });
+  }
+
+  return new Response(results.archive, {
+    headers: {
+      "Content-Disposition": `attachment; filename="${results.name}"`,
+      "Content-Type": "application/gzip",
+    },
+  });
+}
+
+class GenerateBackgroundTaskArtifactArchiveService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const artifact = await this.#prismaClient.backgroundTaskArtifact.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        backgroundTask: true,
+      },
+    });
+
+    if (!artifact) {
+      return;
+    }
+
+    const archive = archiver("tar", {
+      gzip: true,
+      zlib: { level: 9 }, // Sets the compression level
+    });
+
+    function addFileToContext(contents: string, path: string) {
+      // Append files to the archive
+      archive.append(contents, { name: `ctx/${path}` });
+    }
+
+    // Good practice to catch warnings (ie stat failures and other non-blocking errors)
+    archive.on("warning", function (err) {
+      if (err.code === "ENOENT") {
+        // log warning
+        console.warn(err);
+      } else {
+        // throw error
+        throw err;
+      }
+    });
+
+    // Good practice to catch this error explicitly
+    archive.on("error", function (err) {
+      throw err;
+    });
+
+    addFileToContext(artifact.bundle, `src/${artifact.fileName}`);
+    addFileToContext(
+      JSON.stringify(this.#generatePackageJson(artifact, artifact.backgroundTask)),
+      "package.json"
+    );
+    addFileToContext(this.#generateDockerfile(artifact, artifact.backgroundTask), "Dockerfile");
+    addFileToContext(this.#generateIndexJs(artifact, artifact.backgroundTask), "src/index.js");
+
+    // Finalize the archive
+    archive.finalize();
+
+    return {
+      archive,
+      name: `${artifact.id}.tar.gz`,
+    };
+  }
+
+  #generatePackageJson(artifact: BackgroundTaskArtifact, task: BackgroundTask) {
+    return {
+      name: task.slug,
+      version: artifact.version,
+      description: `Trigger background task ${task.slug}`,
+      main: "src/index.js",
+      scripts: {
+        start: "node src/index.js",
+      },
+      dependencies: artifact.dependencies,
+      engines: {
+        node: this.#getNodeVersion(artifact.nodeVersion),
+      },
+    };
+  }
+
+  #generateDockerfile(artifact: BackgroundTaskArtifact, task: BackgroundTask) {
+    return `FROM amd64/node:${this.#getNodeVersion(artifact.nodeVersion)}-bullseye-slim
+    
+    WORKDIR /usr/src/app
+    
+    COPY ctx/package*.json ./
+    
+    RUN npm install
+    
+    COPY ctx/. .
+    
+    CMD [ "npm", "start" ]
+    `;
+  }
+
+  #generateIndexJs(artifact: BackgroundTaskArtifact, task: BackgroundTask) {
+    return `
+      const task = require("./${artifact.fileName}").default; 
+      console.log(task);
+      console.log(process.env);
+    `;
+  }
+
+  // replace the v if it exists
+  #getNodeVersion(version: string) {
+    return version.replace("v", "");
+  }
+}

apps/webapp/app/routes/api.v1.background.tasks.ts

@@ -0,0 +1,60 @@
+import type { ActionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { DeployBackgroundTaskRequestBodySchema } from "@trigger.dev/core";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { DeployBackgroundTaskService } from "~/services/backgroundTasks/deployBackgroundTask.server";
+
+export async function action({ request, params }: ActionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  const body = DeployBackgroundTaskRequestBodySchema.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new DeployBackgroundTaskService();
+
+  try {
+    const results = await service.call(authenticationResult.environment, body.data);
+
+    if (!results) {
+      return json(
+        {
+          error: `Unable to deploy background task, Task with ID = ${body.data.id} not found`,
+        },
+        { status: 500 }
+      );
+    }
+
+    const { artifact, imageConfig } = results;
+
+    return json({
+      id: artifact.id,
+      hash: artifact.hash,
+      image: imageConfig.image,
+      tag: imageConfig.tag,
+      createdAt: artifact.createdAt,
+      updatedAt: artifact.updatedAt,
+    });
+  } catch (error) {
+    if (error instanceof Error) {
+      return json({ error: error.message }, { status: 400 });
+    }
+
+    return json({ error: "Something went wrong" }, { status: 500 });
+  }
+}

apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier.ts

@@ -1,4 +1,5 @@
 import { ActionArgs, LoaderArgs, json } from "@remix-run/server-runtime";
+import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { z } from "zod";
 import { PrismaClient, prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
@@ -99,6 +100,9 @@ export class TriggerEndpointIndexHookService {
           slug: endpointSlug,
         },
       },
+      include: {
+        environment: true,
+      },
     });
 
     if (!endpoint) {
@@ -122,6 +126,8 @@ export class TriggerEndpointIndexHookService {
       },
       {
         runAt: new Date(Date.now() + 5000),
+        maxAttempts:
+          endpoint.environment.type === RuntimeEnvironmentType.DEVELOPMENT ? 1 : undefined,
       }
     );
   }

apps/webapp/app/routes/api.v1.events.$eventId.cancel.ts

@@ -0,0 +1,52 @@
+import type { ActionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { eventRecordToApiJson } from "~/api.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CancelEventService } from "~/services/events/cancelEvent.server";
+import { logger } from "~/services/logger.server";
+
+const ParamsSchema = z.object({
+  eventId: z.string(),
+});
+
+export async function action({ request, params }: ActionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  const parsed = ParamsSchema.safeParse(params);
+
+  if (!parsed.success) {
+    return json({ error: "Invalid or Missing eventId" }, { status: 400 });
+  }
+
+  const { eventId } = parsed.data;
+
+  const service = new CancelEventService();
+  try {
+    const updatedEvent = await service.call(authenticatedEnv, eventId);
+
+    if (!updatedEvent) {
+      return json({ error: "Event not found" }, { status: 404 });
+    }
+
+    return json(eventRecordToApiJson(updatedEvent));
+  } catch (err) {
+    logger.error("CancelEventService.call() error", {
+      error: err,
+    });
+
+    return json({ error: "Internal Server Error" }, { status: 500 });
+  }
+}

apps/webapp/app/routes/api.v1.events.$eventId.ts

@@ -1,6 +1,6 @@
-import type { ActionArgs, LoaderArgs } from "@remix-run/server-runtime";
+import type { LoaderArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import { cors } from "remix-utils";
+import { GetEvent } from "@trigger.dev/core";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
@@ -32,9 +32,36 @@ export async function loader({ request, params }: LoaderArgs) {
 
   const { eventId } = parsed.data;
 
-  const event = await prisma.eventRecord.findFirst({
+  const event = await findEventRecord(eventId, authenticatedEnv.id);
+
+  if (!event) {
+    return apiCors(request, json({ error: "Event not found" }, { status: 404 }));
+  }
+
+  return apiCors(request, json(toJSON(event)));
+}
+
+function toJSON(eventRecord: FoundEventRecord): GetEvent {
+  return {
+    id: eventRecord.eventId,
+    name: eventRecord.name,
+    createdAt: eventRecord.createdAt,
+    updatedAt: eventRecord.updatedAt,
+    runs: eventRecord.runs.map((run) => ({
+      id: run.id,
+      status: run.status,
+      startedAt: run.startedAt,
+      completedAt: run.completedAt,
+    })),
+  };
+}
+
+type FoundEventRecord = NonNullable<Awaited<ReturnType<typeof findEventRecord>>>;
+
+async function findEventRecord(eventId: string, environmentId: string) {
+  return await prisma.eventRecord.findUnique({
     select: {
-      id: true,
+      eventId: true,
       name: true,
       createdAt: true,
       updatedAt: true,
@@ -48,14 +75,10 @@ export async function loader({ request, params }: LoaderArgs) {
       },
     },
     where: {
-      id: eventId,
-      environmentId: authenticatedEnv.id,
+      eventId_environmentId: {
+        eventId,
+        environmentId,
+      },
     },
   });
-
-  if (!event) {
-    return apiCors(request, json({ error: "Event not found" }, { status: 404 }));
-  }
-
-  return apiCors(request, json(event));
 }

apps/webapp/app/routes/api.v1.events.ts

@@ -4,6 +4,7 @@ import { SendEventBodySchema } from "@trigger.dev/core";
 import { generateErrorMessage } from "zod-error";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { IngestSendEvent } from "~/services/events/ingestSendEvent.server";
+import { eventRecordToApiJson } from "~/api.server";
 
 export async function action({ request }: ActionArgs) {
   // Ensure this is a POST request
@@ -33,5 +34,9 @@ export async function action({ request }: ActionArgs) {
 
   const event = await service.call(authenticatedEnv, body.data.event, body.data.options);
 
-  return json(event);
+  if (!event) {
+    return json({ error: "Failed to create event" }, { status: 500 });
+  }
+
+  return json(eventRecordToApiJson(event));
 }

apps/webapp/app/routes/api.v1.runs.$runId.tasks.ts

@@ -107,6 +107,32 @@ export class RunTaskService {
       });
 
       if (existingTask) {
+        if (existingTask.status === "CANCELED") {
+          const existingTaskStatus =
+            (taskBody.delayUntil && taskBody.delayUntil.getTime() > Date.now()) || taskBody.trigger
+              ? "WAITING"
+              : taskBody.noop
+              ? "COMPLETED"
+              : "RUNNING";
+
+          const resumedExistingTask = await tx.task.update({
+            where: {
+              id: existingTask.id,
+            },
+            data: {
+              status: existingTaskStatus,
+              startedAt: new Date(),
+              completedAt: existingTaskStatus === "COMPLETED" ? new Date() : undefined,
+            },
+            include: {
+              run: true,
+              attempts: true,
+            },
+          });
+
+          return resumedExistingTask;
+        }
+
         return existingTask;
       }
 
@@ -157,7 +183,7 @@ export class RunTaskService {
             },
           },
           parent: taskBody.parentId ? { connect: { id: taskBody.parentId } } : undefined,
-          name: taskBody.name,
+          name: taskBody.name ?? "Task",
           description: taskBody.description,
           status,
           startedAt: new Date(),

apps/webapp/app/routes/api.v2.$endpointSlug.sources.$id.ts

@@ -0,0 +1,72 @@
+import type { ActionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { UpdateTriggerSourceBodyV2Schema } from "@trigger.dev/core";
+import { z } from "zod";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { UpdateSourceServiceV2 } from "~/services/sources/updateSourceV2.server";
+
+const ParamsSchema = z.object({
+  endpointSlug: z.string(),
+  id: z.string(),
+});
+
+export async function action({ request, params }: ActionArgs) {
+  logger.info("Updating source", { url: request.url });
+
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "PUT") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    logger.info("Invalid params", { params });
+
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  const body = UpdateTriggerSourceBodyV2Schema.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new UpdateSourceServiceV2();
+
+  try {
+    const source = await service.call({
+      environment: authenticatedEnv,
+      payload: body.data,
+      endpointSlug: parsedParams.data.endpointSlug,
+      id: parsedParams.data.id,
+    });
+
+    return json(source);
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Error activating http source", {
+        url: request.url,
+        error: error.message,
+      });
+
+      return json({ error: error.message }, { status: 400 });
+    }
+
+    return json({ error: "Something went wrong" }, { status: 500 });
+  }
+}

apps/webapp/app/routes/api.v2.$endpointSlug.triggers.$id.registrations.$key.ts

@@ -0,0 +1,106 @@
+import type { ActionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import {
+  REGISTER_SOURCE_EVENT_V2,
+  RegisterSourceEventV2,
+  RegisterTriggerBodySchemaV2,
+} from "@trigger.dev/core";
+import { z } from "zod";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { IngestSendEvent } from "~/services/events/ingestSendEvent.server";
+import { logger } from "~/services/logger.server";
+import { RegisterTriggerSourceServiceV2 } from "~/services/triggers/registerTriggerSourceV2.server";
+
+const ParamsSchema = z.object({
+  endpointSlug: z.string(),
+  id: z.string(),
+  key: z.string(),
+});
+
+export async function action({ request, params }: ActionArgs) {
+  logger.info("Registering trigger", { url: request.url });
+
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "PUT") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    logger.info("Invalid params", { params });
+
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  const body = RegisterTriggerBodySchemaV2.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new RegisterTriggerSourceServiceV2();
+
+  try {
+    const registration = await service.call({
+      environment: authenticatedEnv,
+      payload: body.data,
+      endpointSlug: parsedParams.data.endpointSlug,
+      id: parsedParams.data.id,
+      key: parsedParams.data.key,
+    });
+
+    if (!registration) {
+      return json({ error: "Could not register trigger" }, { status: 500 });
+    }
+
+    //the source is already active
+    if (registration.source.active) {
+      return json(registration);
+    }
+
+    const payload: RegisterSourceEventV2 = {
+      ...registration,
+      dynamicTriggerId: parsedParams.data.id,
+    };
+
+    const ingestEventService = new IngestSendEvent();
+    await ingestEventService.call(
+      authenticatedEnv,
+      {
+        id: registration.id,
+        name: REGISTER_SOURCE_EVENT_V2,
+        source: "trigger.dev",
+        payload,
+      }
+      //todo accountId?
+      // {accountId: body.data.accountId}
+    );
+
+    return json(registration);
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Error registering trigger", {
+        url: request.url,
+        error: error.message,
+      });
+
+      return json({ error: error.message }, { status: 400 });
+    }
+
+    return json({ error: "Something went wrong" }, { status: 500 });
+  }
+}

apps/webapp/app/routes/healthcheck.tsx

@@ -7,11 +7,15 @@ export const loader: LoaderFunction = async ({ request }) => {
 
   try {
     const url = new URL("/", `http://${host}`);
+
+    if (request.headers.get("x-forwarded-proto") === "https") {
+      url.protocol = "https:";
+    }
     // if we can connect to the database and make a simple query
     // and make a HEAD request to ourselves, then we're good.
     await Promise.all([
       prisma.user.count(),
-      fetch(url.toString(), { method: "HEAD" }).then((r) => {
+      fetch(url.href, { method: "HEAD" }).then((r) => {
         if (!r.ok) return Promise.reject(r);
       }),
     ]);

apps/webapp/app/routes/resources.jobs.$jobId.ts

@@ -0,0 +1,74 @@
+import { ActionFunction } from "@remix-run/node";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import {
+  jsonWithErrorMessage,
+  jsonWithSuccessMessage,
+  redirectWithSuccessMessage,
+} from "~/models/message.server";
+import { DeleteJobService } from "~/services/jobs/deleteJob.server";
+import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
+
+const ParamSchema = z.object({
+  jobId: z.string(),
+});
+
+export const action: ActionFunction = async ({ request, params }) => {
+  const { jobId } = ParamSchema.parse(params);
+  const userId = await requireUserId(request);
+
+  // Find the job
+  const job = await prisma.job.findFirst({
+    where: {
+      id: jobId,
+      organization: {
+        members: {
+          some: {
+            userId,
+          },
+        },
+      },
+    },
+  });
+
+  if (!job) {
+    return jsonWithErrorMessage({ ok: false }, request, `Job could not be scheduled for deletion.`);
+  }
+  try {
+    const deleteJobService = new DeleteJobService();
+
+    await deleteJobService.call(job);
+
+    const url = new URL(request.url);
+    const redirectTo = url.searchParams.get("redirectTo");
+
+    logger.debug("Job scheduled for deletion", {
+      url,
+      redirectTo,
+      job,
+    });
+
+    if (typeof redirectTo === "string" && redirectTo.length > 0) {
+      return redirectWithSuccessMessage(
+        redirectTo,
+        request,
+        `Job ${job.slug} has been scheduled for deletion.`
+      );
+    }
+
+    return jsonWithSuccessMessage(
+      { ok: true },
+      request,
+      `Job ${job.slug} has been scheduled for deletion.`
+    );
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+
+    return jsonWithErrorMessage(
+      { ok: false },
+      request,
+      `Job could not be scheduled for deletion: ${message}`
+    );
+  }
+};

apps/webapp/app/routes/resources.runs.$runId.cancel.ts

@@ -2,6 +2,7 @@ import { parse } from "@conform-to/zod";
 import { ActionFunction, json } from "@remix-run/node";
 import { z } from "zod";
 import { redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
 import { CancelRunService } from "~/services/runs/cancelRun.server";
 
 export const cancelSchema = z.object({
@@ -31,7 +32,19 @@ export const action: ActionFunction = async ({ request, params }) => {
       request,
       `Canceled run. Any pending tasks will be canceled.`
     );
-  } catch (error: any) {
-    return json({ errors: { body: error.message } }, { status: 400 });
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to cancel run", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+      });
+      return json({ errors: { body: error.message } }, { status: 400 });
+    } else {
+      logger.error("Failed to cancel run", { error });
+      return json({ errors: { body: "Unknown error" } }, { status: 400 });
+    }
   }
 };

apps/webapp/app/services/apiAuth.server.ts

@@ -26,30 +26,44 @@ export async function authenticateApiRequest(
     return;
   }
 
+  return await authenticateApiKey(result, { allowPublicKey });
+}
+
+export async function authenticateApiKey(
+  apiKey: string,
+  { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
+): Promise<ApiAuthenticationResult | undefined> {
+  const type = isPublicApiKey(apiKey) ? ("PUBLIC" as const) : ("PRIVATE" as const);
+
   //if it's a public API key and we don't allow public keys, return
   if (!allowPublicKey) {
-    const environment = await findEnvironmentByApiKey(result.apiKey);
+    const environment = await findEnvironmentByApiKey(apiKey);
     if (!environment) return;
+
     return {
-      ...result,
+      apiKey,
+      type,
       environment,
     };
   }
 
-  switch (result.type) {
+  switch (type) {
     case "PUBLIC": {
-      const environment = await findEnvironmentByPublicApiKey(result.apiKey);
+      const environment = await findEnvironmentByPublicApiKey(apiKey);
       if (!environment) return;
       return {
-        ...result,
+        apiKey,
+        type,
         environment,
       };
     }
     case "PRIVATE": {
-      const environment = await findEnvironmentByApiKey(result.apiKey);
+      const environment = await findEnvironmentByApiKey(apiKey);
       if (!environment) return;
+
       return {
-        ...result,
+        apiKey,
+        type,
         environment,
       };
     }
@@ -69,6 +83,6 @@ export function getApiKeyFromRequest(request: Request) {
   }
 
   const apiKey = authorization.data.replace(/^Bearer /, "");
-  const type = isPublicApiKey(apiKey) ? ("PUBLIC" as const) : ("PRIVATE" as const);
-  return { apiKey, type };
+
+  return apiKey;
 }

apps/webapp/app/services/backgroundTasks/assignOperationToPool.server.ts

@@ -0,0 +1,66 @@
+import {
+  BackgroundTaskImage,
+  BackgroundTaskOperation,
+  BackgroundTaskVersion,
+} from "@trigger.dev/database";
+import type { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
+import { $transaction, prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { AutoScalePoolService } from "./autoScalePool.server";
+
+export class AssignOperationToPoolService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    operation: BackgroundTaskOperation,
+    version: BackgroundTaskVersion,
+    image: BackgroundTaskImage
+  ) {
+    return await $transaction(this.#prismaClient, async (tx) => {
+      const pool = await tx.backgroundTaskMachinePool.upsert({
+        where: {
+          backgroundTaskVersionId_imageId: {
+            backgroundTaskVersionId: version.id,
+            imageId: image.id,
+          },
+        },
+        create: {
+          backgroundTaskVersionId: version.id,
+          imageId: image.id,
+          backgroundTaskId: operation.backgroundTaskId,
+          provider: image.provider,
+          region: version.region,
+          cpu: version.cpu,
+          memory: version.memory,
+          concurrency: version.concurrency,
+          diskSize: version.diskSize,
+        },
+        update: {
+          region: version.region,
+          cpu: version.cpu,
+          memory: version.memory,
+          concurrency: version.concurrency,
+          diskSize: version.diskSize,
+        },
+      });
+
+      const updatedOperation = await tx.backgroundTaskOperation.update({
+        where: {
+          id: operation.id,
+        },
+        data: {
+          status: "ASSIGNED_TO_POOL",
+          poolId: pool.id,
+        },
+      });
+
+      await AutoScalePoolService.enqueue(pool, tx, true);
+
+      return updatedOperation;
+    });
+  }
+}

apps/webapp/app/services/backgroundTasks/autoScalePool.server.ts

@@ -0,0 +1,191 @@
+import {
+  BackgroundTask,
+  BackgroundTaskMachine,
+  BackgroundTaskMachinePool,
+} from "@trigger.dev/database";
+import type { PrismaClientOrTransaction } from "~/db.server";
+import { prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { backgroundTaskProvider } from "./provider.server";
+import { CreateExternalMachineService } from "./createExternalMachine.server";
+
+const frequency = 1000 * 30; // 30 seconds
+
+export class AutoScalePoolService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const pool = await this.#prismaClient.backgroundTaskMachinePool.findUniqueOrThrow({
+      where: {
+        id,
+      },
+      include: {
+        _count: {
+          select: {
+            operations: {
+              where: {
+                status: "ASSIGNED_TO_POOL",
+              },
+            },
+          },
+        },
+        machines: true,
+        backgroundTask: true,
+        backgroundTaskVersion: true,
+      },
+    });
+
+    // Index the machines to gather the current state of the pool
+
+    // If there are no operations, we can re-enqueue this job
+    if (pool._count.operations === 0) {
+      return await AutoScalePoolService.enqueue(pool, this.#prismaClient);
+    }
+
+    // TODO: we probably should just list all the machines for the app?
+
+    const machines = await this.#autoScaleMachines(
+      pool.backgroundTaskVersion.concurrency,
+      pool,
+      pool.machines,
+      pool.backgroundTask
+    );
+
+    // We need to create any pending machines
+    const pendingMachines = machines.filter((machine) => machine.status === "PENDING");
+
+    for (const pendingMachine of pendingMachines) {
+      await CreateExternalMachineService.enqueue(pendingMachine, this.#prismaClient);
+    }
+
+    await backgroundTaskProvider.cleanupForTask(pool.backgroundTask);
+
+    await AutoScalePoolService.enqueue(pool, this.#prismaClient);
+  }
+
+  // If there are operations, we need to scale up the pool
+  // Machines will automatically be restarted when they are returned to the pool
+  // So we just need to make sure at least one machine is running
+  // And if there is not, we need to start one
+  // Status
+  // machine statutes:
+  // PENDING - The record has been created, but the machine has not on the provider
+  // CREATED - The machine has been created on the provider
+  // STARTING
+  // STARTED - The machine is running
+  // STOPPING
+  // STOPPED
+  // DESTROYING
+  // DESTROYED - The machine has been destroyed on the provider
+  // REPLACING - The machine config is being updated on the provider
+  async #autoScaleMachines(
+    target: number,
+    pool: BackgroundTaskMachinePool,
+    existingMachines: BackgroundTaskMachine[],
+    task: BackgroundTask
+  ): Promise<BackgroundTaskMachine[]> {
+    const pendingMachines: BackgroundTaskMachine[] = [];
+
+    // We need to update the pool to have the correct number of machines
+    if (existingMachines.length < target) {
+      const machinesToCreate = target - existingMachines.length;
+
+      for (let i = 0; i < machinesToCreate; i++) {
+        const pendingMachine = await this.#prismaClient.backgroundTaskMachine.create({
+          data: {
+            provider: backgroundTaskProvider.name,
+            poolId: pool.id,
+            backgroundTaskId: pool.backgroundTaskId,
+            backgroundTaskVersionId: pool.backgroundTaskVersionId,
+            backgroundTaskImageId: pool.imageId,
+          },
+        });
+
+        pendingMachines.push(pendingMachine);
+      }
+    }
+
+    const updatedExistingMachines = (
+      await Promise.all(existingMachines.map(async (machine) => this.#indexMachine(machine, task)))
+    ).filter(Boolean);
+
+    const replacedMachines: BackgroundTaskMachine[] = [];
+
+    if (updatedExistingMachines.length < existingMachines.length) {
+      const machinesToReplace = existingMachines.length - updatedExistingMachines.length;
+
+      for (let i = 0; i < machinesToReplace; i++) {
+        const pendingMachine = await this.#prismaClient.backgroundTaskMachine.create({
+          data: {
+            provider: backgroundTaskProvider.name,
+            poolId: pool.id,
+            backgroundTaskId: pool.backgroundTaskId,
+            backgroundTaskVersionId: pool.backgroundTaskVersionId,
+            backgroundTaskImageId: pool.imageId,
+          },
+        });
+
+        replacedMachines.push(pendingMachine);
+      }
+    }
+
+    return [...pendingMachines, ...updatedExistingMachines, ...replacedMachines];
+  }
+
+  async #indexMachine(
+    machine: BackgroundTaskMachine,
+    task: BackgroundTask
+  ): Promise<BackgroundTaskMachine | undefined> {
+    // Using the provider get updated information about the machine (if it has an externalId)
+    if (!machine.externalId) {
+      return machine;
+    }
+
+    const externalMachine = await backgroundTaskProvider.getMachineForTask(
+      machine.externalId,
+      task
+    );
+
+    if (!externalMachine) {
+      await this.#prismaClient.backgroundTaskMachine.delete({
+        where: {
+          id: machine.id,
+        },
+      });
+
+      return;
+    }
+
+    return await this.#prismaClient.backgroundTaskMachine.update({
+      where: {
+        id: machine.id,
+      },
+      data: {
+        status: externalMachine.status,
+        data: externalMachine.data,
+      },
+    });
+  }
+
+  static async enqueue(
+    pool: BackgroundTaskMachinePool,
+    tx: PrismaClientOrTransaction = prisma,
+    force = false
+  ) {
+    return await workerQueue.enqueue(
+      "autoScalePool",
+      {
+        id: pool.id,
+      },
+      {
+        tx,
+        jobKey: `scale:${pool.id}`,
+        runAt: force ? new Date() : new Date(Date.now() + frequency),
+      }
+    );
+  }
+}

apps/webapp/app/services/backgroundTasks/createBackgroundTaskImage.server.ts

@@ -0,0 +1,62 @@
+import { CreateBackgroundTaskImageRequestBody } from "@trigger.dev/core";
+import { PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+import { backgroundTaskProvider } from "./provider.server";
+
+export class CreateBackgroundTaskImageService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    environment: AuthenticatedEnvironment,
+    id: string,
+    payload: CreateBackgroundTaskImageRequestBody
+  ) {
+    // Find the artifact
+    const artifact = await this.#prismaClient.backgroundTaskArtifact.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        backgroundTask: true,
+      },
+    });
+
+    if (!artifact) {
+      return;
+    }
+
+    if (artifact.backgroundTask.projectId !== environment.projectId) {
+      return;
+    }
+
+    const image = await this.#prismaClient.backgroundTaskImage.upsert({
+      where: {
+        backgroundTaskArtifactId_digest: {
+          backgroundTaskArtifactId: artifact.id,
+          digest: payload.digest,
+        },
+      },
+      create: {
+        backgroundTaskArtifactId: artifact.id,
+        backgroundTaskId: artifact.backgroundTaskId,
+        digest: payload.digest,
+        name: payload.name,
+        tag: payload.tag,
+        size: payload.size,
+        provider: backgroundTaskProvider.name,
+      },
+      update: {
+        name: payload.name,
+        tag: payload.tag,
+        size: payload.size,
+      },
+    });
+
+    return image;
+  }
+}

apps/webapp/app/services/backgroundTasks/createExternalMachine.server.ts

@@ -0,0 +1,98 @@
+import { BackgroundTaskMachine } from "@trigger.dev/database";
+import type { PrismaClientOrTransaction } from "~/db.server";
+import { prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { backgroundTaskProvider } from "./provider.server";
+import { resolveBackgroundTaskSecret } from "~/models/backgroundTaskSecret.server";
+import { env } from "~/env.server";
+import { ExternalMachineConfig } from "./providers/types";
+
+export class CreateExternalMachineService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const machine = await this.#prismaClient.backgroundTaskMachine.findUniqueOrThrow({
+      where: {
+        id,
+      },
+      include: {
+        pool: {
+          include: {
+            image: true,
+            backgroundTask: true,
+            backgroundTaskVersion: {
+              include: {
+                environment: true,
+                secrets: {
+                  include: {
+                    secretReference: true,
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    const envVars: Record<string, string> = {};
+
+    for (const secret of machine.pool.backgroundTaskVersion.secrets) {
+      const secretValue = await resolveBackgroundTaskSecret(secret.secretReference);
+
+      if (!secretValue) {
+        continue;
+      }
+
+      envVars[secret.key] = secretValue;
+    }
+
+    envVars["TRIGGER_API_KEY"] = machine.pool.backgroundTaskVersion.environment.apiKey;
+    envVars["TRIGGER_API_URL"] = env.APP_ORIGIN;
+    envVars["TRIGGER_POOL_ID"] = machine.pool.id;
+    envVars["TRIGGER_MACHINE_ID"] = machine.id;
+
+    const config: ExternalMachineConfig = {
+      cpus: machine.pool.cpu,
+      memory: machine.pool.memory,
+      diskSize: machine.pool.diskSize,
+      region: machine.pool.region,
+      env: envVars,
+      image: `${backgroundTaskProvider.registry}/${machine.pool.image.name}:${machine.pool.image.tag}@${machine.pool.image.digest}`,
+    };
+
+    const externalMachine = await backgroundTaskProvider.createMachineForTask(
+      machine.id,
+      machine.pool.backgroundTask,
+      config
+    );
+
+    await this.#prismaClient.backgroundTaskMachine.update({
+      where: {
+        id: machine.id,
+      },
+      data: {
+        externalId: externalMachine.id,
+        status: externalMachine.status,
+        data: externalMachine.data,
+      },
+    });
+  }
+
+  static async enqueue(machine: BackgroundTaskMachine, tx: PrismaClientOrTransaction = prisma) {
+    return await workerQueue.enqueue(
+      "createExternalMachine",
+      {
+        id: machine.id,
+      },
+      {
+        tx,
+        jobKey: `createMachine:${machine.id}`,
+      }
+    );
+  }
+}

apps/webapp/app/services/backgroundTasks/deployBackgroundTask.server.ts

@@ -0,0 +1,86 @@
+import { DeployBackgroundTaskRequestBody } from "@trigger.dev/core";
+import { PrismaClient } from "@trigger.dev/database";
+import nodeCrypto from "node:crypto";
+import { prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+import { backgroundTaskProvider } from "./provider.server";
+
+export class DeployBackgroundTaskService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    environment: AuthenticatedEnvironment,
+    payload: DeployBackgroundTaskRequestBody
+  ) {
+    const hash = this.#hashPayload(payload);
+
+    const backgroundTask = await this.#prismaClient.backgroundTask.findUnique({
+      where: {
+        projectId_slug: {
+          projectId: environment.projectId,
+          slug: payload.id,
+        },
+      },
+    });
+
+    if (!backgroundTask) {
+      return;
+    }
+
+    const artifact = await this.#prismaClient.backgroundTaskArtifact.upsert({
+      where: {
+        backgroundTaskId_version_hash: {
+          backgroundTaskId: backgroundTask.id,
+          version: payload.version,
+          hash,
+        },
+      },
+      create: {
+        backgroundTaskId: backgroundTask.id,
+        fileName: payload.fileName,
+        version: payload.version,
+        hash,
+        bundle: payload.bundle,
+        nodeVersion: payload.nodeVersion,
+        dependencies: payload.dependencies,
+        sourcemap: payload.sourcemap,
+      },
+      update: {
+        fileName: payload.fileName,
+        bundle: payload.bundle,
+        nodeVersion: payload.nodeVersion,
+        dependencies: payload.dependencies,
+        sourcemap: payload.sourcemap,
+      },
+    });
+
+    const imageConfig = await backgroundTaskProvider.prepareArtifact(backgroundTask, artifact);
+
+    return {
+      artifact,
+      imageConfig,
+    };
+  }
+
+  #hashPayload(payload: DeployBackgroundTaskRequestBody) {
+    // Create a hash out of the bundle, the nodeVersion, and a determinstically list of dependencies
+    // This will allow us to determine if the bundle has changed
+    const hash = nodeCrypto.createHash("sha256");
+
+    hash.update(payload.bundle);
+    hash.update(payload.nodeVersion);
+
+    const dependencies = Object.keys(payload.dependencies).sort();
+
+    for (const dependency of dependencies) {
+      hash.update(dependency);
+      hash.update(payload.dependencies[dependency]);
+    }
+
+    return hash.digest("hex");
+  }
+}

apps/webapp/app/services/backgroundTasks/disableBackgroundTask.server.ts

@@ -0,0 +1,64 @@
+import type { BackgroundTaskVersion } from "@trigger.dev/database";
+import type { PrismaClient } from "~/db.server";
+import { prisma } from "~/db.server";
+import { ExtendedEndpoint, findEndpoint } from "~/models/endpoint.server";
+import type { AuthenticatedEnvironment } from "../apiAuth.server";
+
+export type DisableBackgroundTaskServiceOptions = {
+  slug: string;
+  version: string;
+};
+
+export class DisableBackgroundTaskService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    endpointIdOrEndpoint: string | ExtendedEndpoint,
+    options: DisableBackgroundTaskServiceOptions
+  ) {
+    const endpoint =
+      typeof endpointIdOrEndpoint === "string"
+        ? await findEndpoint(endpointIdOrEndpoint)
+        : endpointIdOrEndpoint;
+
+    return this.#disableBackgroundTask(endpoint.environment, options);
+  }
+
+  async #disableBackgroundTask(
+    environment: AuthenticatedEnvironment,
+    options: DisableBackgroundTaskServiceOptions
+  ): Promise<BackgroundTaskVersion | undefined> {
+    const backgroundTask = await this.#prismaClient.backgroundTask.findUnique({
+      where: {
+        projectId_slug: {
+          projectId: environment.projectId,
+          slug: options.slug,
+        },
+      },
+    });
+
+    if (!backgroundTask) {
+      return;
+    }
+
+    const backgroundTaskVersion = await this.#prismaClient.backgroundTaskVersion.findUnique({
+      where: {
+        backgroundTaskId_version_environmentId: {
+          backgroundTaskId: backgroundTask.id,
+          version: options.version,
+          environmentId: environment.id,
+        },
+      },
+    });
+
+    if (!backgroundTaskVersion) {
+      return;
+    }
+
+    // TODO: Disable background task
+  }
+}

apps/webapp/app/services/backgroundTasks/executeBackgroundTaskOperation.server.ts

@@ -0,0 +1,55 @@
+import { PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { AssignOperationToPoolService } from "./assignOperationToPool.server";
+
+export class ExecuteBackgroundTaskOperationService {
+  #prismaClient: PrismaClient;
+  #assignOperationToPoolService = new AssignOperationToPoolService();
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const operation = await this.#prismaClient.backgroundTaskOperation.findUniqueOrThrow({
+      where: {
+        id,
+      },
+      include: {
+        backgroundTask: true,
+        backgroundTaskVersion: true,
+      },
+    });
+
+    // Find the BackgroundTaskImage
+    const image = await this.#prismaClient.backgroundTaskImage.findFirst({
+      where: {
+        backgroundTaskId: operation.backgroundTaskId,
+        tag: operation.backgroundTaskVersion.version,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+
+    // If the image is not found, we need to wait for it to be deployed
+    if (!image) {
+      await this.#prismaClient.backgroundTaskOperation.update({
+        where: {
+          id,
+        },
+        data: {
+          status: "WAITING_ON_IMAGE",
+        },
+      });
+
+      return;
+    }
+
+    return await this.#assignOperationToPoolService.call(
+      operation,
+      operation.backgroundTaskVersion,
+      image
+    );
+  }
+}

apps/webapp/app/services/backgroundTasks/initializeBackgroundTaskOperation.server.ts

@@ -0,0 +1,104 @@
+import { BackgroundTaskOperationParamsSchema } from "@trigger.dev/core";
+import { PrismaClient, RuntimeEnvironmentType, Task } from "@trigger.dev/database";
+import { $transaction, prisma } from "~/db.server";
+import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
+import { KitchenSinkTask } from "~/models/task.server";
+import { workerQueue } from "../worker.server";
+
+export class InitializeBackgroundTaskOperationService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(task: KitchenSinkTask) {
+    const params = BackgroundTaskOperationParamsSchema.safeParse(task.params);
+    // We need to create a new background task operation
+
+    if (!params.success) {
+      await this.#resumeTaskWithError(task, params.error.message);
+      return;
+    }
+
+    const backgroundTask = await this.#prismaClient.backgroundTask.findUnique({
+      where: {
+        projectId_slug: {
+          projectId: task.run.projectId,
+          slug: params.data.id,
+        },
+      },
+      include: {
+        versions: {
+          where: {
+            version: params.data.version,
+          },
+        },
+      },
+    });
+
+    if (!backgroundTask) {
+      await this.#resumeTaskWithError(task, `Background task ${params.data.id} not found`);
+      return;
+    }
+
+    const version = backgroundTask.versions[0];
+
+    if (!version) {
+      await this.#resumeTaskWithError(
+        task,
+        `Background task ${params.data.id} version ${params.data.version} not found`
+      );
+      return;
+    }
+
+    await $transaction(this.#prismaClient, async (tx) => {
+      const operation = await tx.backgroundTaskOperation.create({
+        data: {
+          backgroundTaskId: backgroundTask.id,
+          backgroundTaskVersionId: version.id,
+          taskId: task.id,
+          payload: params.data.payload,
+        },
+      });
+
+      await workerQueue.enqueue(
+        "executeBackgroundTaskOperation",
+        {
+          id: operation.id,
+        },
+        { tx }
+      );
+
+      return operation;
+    });
+  }
+
+  async #resumeTaskWithError(task: KitchenSinkTask, message: string) {
+    await $transaction(this.#prismaClient, async (tx) => {
+      await tx.task.update({
+        where: { id: task.id },
+        data: {
+          status: "ERRORED",
+          completedAt: new Date(),
+          output: { message },
+        },
+      });
+
+      await tx.taskAttempt.updateMany({
+        where: {
+          taskId: task.id,
+          status: "PENDING",
+        },
+        data: {
+          status: "ERRORED",
+          error: message,
+        },
+      });
+
+      await enqueueRunExecutionV2(task.run, prisma, {
+        skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
+      });
+    });
+  }
+}

apps/webapp/app/services/backgroundTasks/provider.server.ts

@@ -0,0 +1,56 @@
+import { BackgroundTask, BackgroundTaskProviderStrategy } from "@trigger.dev/database";
+import { env } from "~/env.server";
+import { FlyBackgroundTaskProvider } from "./providers/fly.server";
+import { BackgroundTaskProvider, ExternalMachine, ExternalMachineConfig } from "./providers/types";
+
+export class UnsupportedBackgroundTaskProvider implements BackgroundTaskProvider {
+  async prepareArtifact(task: BackgroundTask): Promise<any> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  get name(): BackgroundTaskProviderStrategy {
+    return "UNSUPPORTED";
+  }
+
+  get defaultRegion(): string {
+    return "UNSUPPORTED";
+  }
+
+  get registry(): string {
+    return "UNSUPPORTED";
+  }
+
+  async getMachineForTask(id: string, task: BackgroundTask): Promise<ExternalMachine | undefined> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  getMachinesForTask(task: BackgroundTask): Promise<Array<ExternalMachine>> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  createMachineForTask(
+    id: string,
+    task: BackgroundTask,
+    config: ExternalMachineConfig
+  ): Promise<ExternalMachine> {
+    throw new Error("Unsupported background task provider");
+  }
+
+  cleanupForTask(task: BackgroundTask): Promise<void> {
+    throw new Error("Unsupported background task provider");
+  }
+}
+
+let backgroundTaskProvider: BackgroundTaskProvider;
+
+if (env.FLY_IO_API_TOKEN && env.FLY_IO_API_URL && env.FLY_IO_ORG_SLUG) {
+  backgroundTaskProvider = new FlyBackgroundTaskProvider(
+    env.FLY_IO_API_URL,
+    env.FLY_IO_ORG_SLUG,
+    env.FLY_IO_API_TOKEN
+  );
+} else {
+  backgroundTaskProvider = new UnsupportedBackgroundTaskProvider();
+}
+
+export { backgroundTaskProvider };

apps/webapp/app/services/backgroundTasks/providers/fly.server.ts

@@ -0,0 +1,569 @@
+import {
+  BackgroundTask,
+  BackgroundTaskArtifact,
+  BackgroundTaskMachineStatus,
+  BackgroundTaskProviderStrategy,
+} from "@trigger.dev/database";
+import { z } from "zod";
+import { logger } from "~/services/logger.server";
+import { ZodResponse, zodfetch } from "~/zodfetch.server";
+import { BackgroundTaskProvider, ExternalMachine, ExternalMachineConfig } from "./types";
+import retry from "async-retry";
+import AsyncRetry from "async-retry";
+
+const FlyAppSchema = z.object({
+  name: z.string(),
+  organization: z.object({
+    name: z.string(),
+    slug: z.string(),
+  }),
+  status: z.string(),
+});
+
+const FlyCreateAppSchema = z.object({
+  app_name: z.string(),
+  org_slug: z.string(),
+  network: z.string().optional(),
+});
+
+const FlyCheckStatusSchema = z.object({
+  name: z.string(),
+  output: z.string(),
+  status: z.string(),
+  updated_at: z.coerce.date().optional(),
+});
+
+const FlyMachineGuestSchema = z.object({
+  cpu_kind: z.enum(["shared", "dedicated"]),
+  cpus: z.number(),
+  memory_mb: z.number(),
+});
+
+const FlyMachineMetricsSchema = z.object({
+  path: z.string(),
+  port: z.number(),
+});
+
+const FlyMachineMountSchema = z.object({
+  encrypted: z.boolean().optional(),
+  name: z.string().optional(),
+  path: z.string(),
+  volume: z.string(),
+  size_gb: z.number().optional(),
+});
+
+const FlyMachineProcessSchema = z.object({
+  cmd: z.array(z.string()).optional(),
+  entrypoint: z.string().optional(),
+  env: z.record(z.string()).default({}),
+  exec: z.array(z.string()).optional(),
+  user: z.string().optional(),
+});
+
+const FlyMachineRestartSchema = z.object({
+  max_retries: z.number().optional(),
+  policy: z.string().optional(),
+});
+
+const FlyMachineHTTPHeaderSchema = z.array(
+  z.object({ name: z.string(), values: z.array(z.string()) })
+);
+
+const FlyMachineCheckSchema = z.object({
+  grace_period: z.string().optional(),
+  headers: z.array(FlyMachineHTTPHeaderSchema).default([]),
+  interval: z.string().optional(),
+  method: z.string().optional(),
+  path: z.string().optional(),
+  port: z.number().optional(),
+  protocol: z.string().optional(),
+  timeouyt: z.string().optional(),
+  tls_server_name: z.string().optional(),
+  tls_skip_verify: z.boolean().optional(),
+  type: z.string().optional(),
+});
+
+const FlyMachineServiceConcurrencySchema = z.object({
+  hard_limit: z.number().optional(),
+  soft_limit: z.number().optional(),
+  type: z.string(),
+});
+
+const FlyHTTPOptionsSchema = z.object({
+  compress: z.boolean().optional(),
+  response: z
+    .object({
+      headers: z.record(z.string()).default({}),
+    })
+    .optional(),
+});
+
+const FlyMachinePortSchema = z.object({
+  end_port: z.number(),
+  force_https: z.boolean().optional(),
+  handlers: z.array(z.string()).default([]),
+  http_options: FlyHTTPOptionsSchema.optional(),
+});
+
+const FlyMachineServiceSchema = z.object({
+  autostart: z.boolean().optional(),
+  autostop: z.boolean().optional(),
+  checks: z.array(FlyMachineCheckSchema).default([]),
+  concurrency: FlyMachineServiceConcurrencySchema,
+  force_instance_description: z.string().optional(),
+  force_instance_key: z.string().optional(),
+  internal_port: z.number().optional(),
+  min_machines_running: z.number().optional(),
+  ports: z.array(FlyMachinePortSchema).default([]),
+  protocol: z.string().optional(),
+});
+
+const FlyMachineStaticSchema = z.object({
+  guest_path: z.string(),
+  url_prefix: z.string(),
+});
+
+const FlyMachineStopConfigSchema = z.object({
+  signal: z.string().optional(),
+  timeout: z.string().optional(),
+});
+
+const FlyMachineConfigSchema = z.object({
+  auto_destroy: z.boolean().optional(),
+  env: z.record(z.string()).default({}),
+  checks: z.record(FlyMachineCheckSchema).default({}),
+  metadata: z.record(z.string()).default({}),
+  guest: FlyMachineGuestSchema,
+  image: z.string(),
+  metrics: FlyMachineMetricsSchema.optional(),
+  mounts: z.array(FlyMachineMountSchema).default([]),
+  processes: z.array(FlyMachineProcessSchema).default([]),
+  restart: FlyMachineRestartSchema.default({}),
+  services: z.array(FlyMachineServiceSchema).default([]),
+  standbys: z.array(z.string()).default([]),
+  statics: z.array(FlyMachineStaticSchema).default([]),
+  stop_config: FlyMachineStopConfigSchema.optional(),
+});
+
+const FlyMachineImageRefSchema = z.object({
+  digest: z.string(),
+  registry: z.string(),
+  repository: z.string(),
+  tag: z.string(),
+  labels: z.record(z.string()).nullable().default({}),
+});
+
+const FlyMachineStateSchema = z.enum([
+  "created",
+  "starting",
+  "started",
+  "stopping",
+  "stopped",
+  "destroying",
+  "destroyed",
+  "replacing",
+]);
+
+const FlyMachineEventSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  status: z.string(),
+  source: z.string(),
+  timestamp: z.coerce.date(),
+  request: z.any(),
+});
+
+const FlyMachineSchema = z.object({
+  id: z.string(),
+  instance_id: z.string(),
+  name: z.string(),
+  nonce: z.string().optional(),
+  private_ip: z.string(),
+  region: z.string(),
+  state: FlyMachineStateSchema,
+  config: FlyMachineConfigSchema,
+  checks: z.array(FlyCheckStatusSchema).optional(),
+  events: z.array(FlyMachineEventSchema).default([]),
+  image_ref: FlyMachineImageRefSchema,
+  created_at: z.coerce.date(),
+  updated_at: z.coerce.date().optional(),
+});
+
+const FlyVolumeSchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  state: z.string(),
+  region: z.string(),
+  size_gb: z.number(),
+  encrypted: z.boolean(),
+  created_at: z.coerce.date(),
+  attached_machine_id: z.string().nullable().optional(),
+  attached_alloc_id: z.string().nullable().optional(),
+  blocks: z.number(),
+  block_size: z.number(),
+  blocks_free: z.number(),
+  blocks_avail: z.number(),
+  fstype: z.string(),
+  host_dedication_key: z.string().nullable().optional(),
+});
+
+const FlyCreateVolumeSchema = z.object({
+  name: z.string(),
+  region: z.string(),
+  size_gb: z.number(),
+  machines_only: z.boolean().optional(),
+  encrypted: z.boolean().optional(),
+});
+
+const FlyCreateMachineSchema = z.object({
+  name: z.string(),
+  lease_ttl: z.number().optional(),
+  region: z.string(),
+  config: FlyMachineConfigSchema,
+  skip_launch: z.boolean().optional(),
+  skip_service_registration: z.boolean().optional(),
+});
+
+export class FlyBackgroundTaskProvider implements BackgroundTaskProvider {
+  private readonly _logger = logger.child("FlyBackgroundTaskProvider");
+
+  get name(): BackgroundTaskProviderStrategy {
+    return "FLY_IO";
+  }
+
+  get registry(): string {
+    return "registry.fly.io";
+  }
+
+  constructor(
+    private readonly url: string,
+    private readonly org: string,
+    private readonly token: string
+  ) {}
+
+  get defaultRegion(): string {
+    return "iad";
+  }
+
+  async prepareArtifact(
+    task: BackgroundTask,
+    artifact: BackgroundTaskArtifact
+  ): Promise<{ image: string; tag: string }> {
+    // Check that the app has been created
+    const app = await this.#getApp(this.#appNameForTask(task));
+
+    if (app) {
+      return {
+        image: app.name,
+        tag: artifact.version,
+      };
+    }
+
+    // Create the app
+    const created = await this.#createApp({
+      app_name: this.#appNameForTask(task),
+      network: this.#networkNameForTask(task),
+      org_slug: this.org,
+    });
+
+    if (!created) {
+      throw new Error("Failed to create app");
+    }
+
+    return {
+      image: this.#appNameForTask(task),
+      tag: artifact.version,
+    };
+  }
+
+  async getMachineForTask(id: string, task: BackgroundTask): Promise<ExternalMachine | undefined> {
+    const response = await this.#fetch(
+      FlyMachineSchema,
+      `/v1/apps/${this.#appNameForTask(task)}/machines/${id}`
+    );
+
+    if (!response.ok) {
+      return;
+    }
+
+    return this.#flyMachineToExternalMachine(response.data);
+  }
+
+  async getMachinesForTask(task: BackgroundTask): Promise<Array<ExternalMachine>> {
+    const response = await this.#fetch(
+      z.array(FlyMachineSchema),
+      `/v1/apps/${this.#appNameForTask(task)}/machines`
+    );
+
+    if (!response.ok) {
+      return [];
+    }
+
+    return response.data.map((machine) => this.#flyMachineToExternalMachine(machine));
+  }
+
+  async createMachineForTask(
+    id: string,
+    task: BackgroundTask,
+    config: ExternalMachineConfig
+  ): Promise<ExternalMachine> {
+    // We have to create a volume first
+    const volume = await this.#createVolume(this.#appNameForTask(task), {
+      name: id,
+      region: config.region,
+      size_gb: config.diskSize,
+      encrypted: true,
+      machines_only: true,
+    });
+
+    const machine = await this.#createMachine(this.#appNameForTask(task), {
+      name: id,
+      region: config.region,
+      config: {
+        image: config.image,
+        env: config.env,
+        guest: {
+          cpu_kind: "shared",
+          cpus: config.cpus,
+          memory_mb: config.memory,
+        },
+        auto_destroy: false,
+        mounts: [
+          {
+            volume: volume.id,
+            path: "/data",
+          },
+        ],
+      },
+    });
+
+    return this.#flyMachineToExternalMachine(machine);
+  }
+
+  async cleanupForTask(task: BackgroundTask): Promise<void> {
+    const volumes = await this.#listVolumes(this.#appNameForTask(task));
+
+    if (!volumes) {
+      return;
+    }
+
+    // Destroy any volumn created more than 30 seconds ago that doesn't have a machine attached
+    const destroyableVolumes = volumes.filter(
+      (volume) =>
+        volume.created_at.getTime() < Date.now() - 30 * 1000 &&
+        !volume.attached_machine_id &&
+        !volume.attached_alloc_id &&
+        volume.state !== "pending_destroy"
+    );
+
+    this._logger.debug("cleanupForTask", {
+      volumesToDestroy: destroyableVolumes.length,
+    });
+
+    for (const volume of destroyableVolumes) {
+      await this.#destroyVolume(this.#appNameForTask(task), volume.id);
+    }
+  }
+
+  #flyMachineToExternalMachine(flyMachine: z.output<typeof FlyMachineSchema>): ExternalMachine {
+    return {
+      id: flyMachine.id,
+      status: this.#flyStateToStatus(flyMachine.state),
+      data: flyMachine,
+    };
+  }
+
+  #flyStateToStatus(state: z.infer<typeof FlyMachineStateSchema>): BackgroundTaskMachineStatus {
+    const mappings: Record<z.infer<typeof FlyMachineStateSchema>, BackgroundTaskMachineStatus> = {
+      created: "CREATED",
+      starting: "STARTING",
+      started: "STARTED",
+      stopping: "STOPPING",
+      stopped: "STOPPED",
+      destroying: "DESTROYING",
+      destroyed: "DESTROYED",
+      replacing: "REPLACING",
+    };
+
+    return mappings[state];
+  }
+
+  async #getApp(appName: string) {
+    const response = await this.#fetch(FlyAppSchema, `/v1/apps/${appName}`);
+
+    if (!response.ok) {
+      return;
+    }
+
+    return response.data;
+  }
+
+  async #createApp(body: z.input<typeof FlyCreateAppSchema>) {
+    const response = await this.#fetch(z.any(), "/v1/apps", {
+      method: "POST",
+      body: JSON.stringify(body),
+    });
+
+    return response.ok;
+  }
+
+  async #createMachine(
+    appName: string,
+    body: z.input<typeof FlyCreateMachineSchema>
+  ): Promise<z.output<typeof FlyMachineSchema>> {
+    const response = await this.#fetch(
+      FlyMachineSchema,
+      `/v1/apps/${appName}/machines`,
+      {
+        method: "POST",
+        body: JSON.stringify(body),
+      },
+      {
+        retries: 5,
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error("Failed to create machine");
+    }
+
+    return response.data;
+  }
+
+  async #createVolume(
+    appName: string,
+    body: z.input<typeof FlyCreateVolumeSchema>
+  ): Promise<z.output<typeof FlyVolumeSchema>> {
+    const response = await this.#fetch(
+      FlyVolumeSchema,
+      `/v1/apps/${appName}/volumes`,
+      {
+        method: "POST",
+        body: JSON.stringify(body),
+      },
+      {
+        retries: 5,
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error("Failed to create volume");
+    }
+
+    return response.data;
+  }
+
+  async #listVolumes(
+    appName: string
+  ): Promise<Array<z.output<typeof FlyVolumeSchema>> | undefined> {
+    const response = await this.#fetch(z.array(FlyVolumeSchema), `/v1/apps/${appName}/volumes`, {
+      method: "GET",
+    });
+
+    if (!response.ok) {
+      return;
+    }
+
+    return response.data;
+  }
+
+  async #destroyVolume(appName: string, id: string): Promise<boolean> {
+    const response = await this.#fetch(z.any(), `/v1/apps/${appName}/volumes/${id}`, {
+      method: "DELETE",
+    });
+
+    return response.ok;
+  }
+
+  async #fetch<TResponseSchema extends z.ZodTypeAny>(
+    schema: TResponseSchema,
+    path: string,
+    requestInit?: RequestInit,
+    retryOptions?: AsyncRetry.Options
+  ): Promise<ZodResponse<TResponseSchema>> {
+    const headers = new Headers(requestInit?.headers ?? {});
+
+    // Add the common headers
+    headers.set("Authorization", `Bearer ${this.token}`);
+    headers.set("Accept", "application/json");
+    headers.set("User-Agent", "Trigger.dev/2.1.0");
+
+    if (requestInit?.body) {
+      headers.set("Content-Type", "application/json; charset=utf-8");
+    }
+
+    if (retryOptions) {
+      return await retry(
+        async (bail) => {
+          const response = await zodfetch(schema, `${this.url}${path}`, {
+            ...requestInit,
+            headers,
+          });
+
+          if (!response.ok) {
+            if (response.status === 429 || response.status >= 500) {
+              throw new Error(
+                `[${response.status}] Request ${
+                  requestInit?.method ?? "GET"
+                } ${path} failed: ${JSON.stringify(response.error)}`
+              );
+            }
+
+            bail(
+              new Error(
+                `[${response.status}] Request ${
+                  requestInit?.method ?? "GET"
+                } ${path} failed: ${JSON.stringify(response.error)}`
+              )
+            );
+            return response;
+          }
+
+          return response;
+        },
+        {
+          ...retryOptions,
+          onRetry: (e, attempt) => {
+            this._logger.debug("fetch.retry", {
+              url: `${this.url}${path}`,
+              attempt,
+              response: {
+                ok: response.ok,
+                status: response.status,
+                error: response.ok ? undefined : response.error,
+                err: {
+                  message: e.message,
+                  stack: e.stack,
+                },
+              },
+            });
+          },
+        }
+      );
+    }
+
+    const response = await zodfetch(schema, `${this.url}${path}`, {
+      ...requestInit,
+      headers,
+    });
+
+    this._logger.debug("fetch", {
+      url: `${this.url}${path}`,
+      response: {
+        ok: response.ok,
+        status: response.status,
+        error: response.ok ? undefined : response.error,
+      },
+    });
+
+    return response;
+  }
+
+  #appNameForTask(task: BackgroundTask): string {
+    return `${task.id}-${task.slug}`;
+  }
+
+  #networkNameForTask(task: BackgroundTask): string {
+    return task.projectId;
+  }
+}

apps/webapp/app/services/backgroundTasks/providers/types.ts

@@ -0,0 +1,43 @@
+import type {
+  BackgroundTask,
+  BackgroundTaskArtifact,
+  BackgroundTaskMachineStatus,
+  BackgroundTaskProviderStrategy,
+} from "@trigger.dev/database";
+
+export type ExternalMachine = {
+  id: string;
+  status: BackgroundTaskMachineStatus;
+  data: any;
+};
+
+export type ExternalMachineConfig = {
+  cpus: number;
+  memory: number;
+  diskSize: number;
+  region: string;
+  image: string;
+  env: Record<string, string>;
+};
+
+export interface BackgroundTaskProvider {
+  prepareArtifact(
+    task: BackgroundTask,
+    artifact: BackgroundTaskArtifact
+  ): Promise<{ image: string; tag: string }>;
+
+  get name(): BackgroundTaskProviderStrategy;
+  get defaultRegion(): string;
+  get registry(): string;
+
+  getMachineForTask(id: string, task: BackgroundTask): Promise<ExternalMachine | undefined>;
+  getMachinesForTask(task: BackgroundTask): Promise<Array<ExternalMachine>>;
+
+  createMachineForTask(
+    id: string,
+    task: BackgroundTask,
+    config: ExternalMachineConfig
+  ): Promise<ExternalMachine>;
+
+  cleanupForTask(task: BackgroundTask): Promise<void>;
+}

apps/webapp/app/services/backgroundTasks/registerBackgroundTask.server.ts

@@ -0,0 +1,221 @@
+import { BackgroundTaskMetadata } from "@trigger.dev/core";
+import type { BackgroundTaskVersion, Endpoint } from "@trigger.dev/database";
+import { DEFAULT_MAX_CONCURRENT_RUNS } from "~/consts";
+import type { PrismaClient } from "~/db.server";
+import { prisma } from "~/db.server";
+import {
+  createBackgroundTaskSecret,
+  deleteBackgroundTaskSecret,
+  updateBackgroundTaskSecret,
+} from "~/models/backgroundTaskSecret.server";
+import { ExtendedEndpoint, findEndpoint } from "~/models/endpoint.server";
+import type { AuthenticatedEnvironment } from "../apiAuth.server";
+import { backgroundTaskProvider } from "./provider.server";
+
+export class RegisterBackgroundTaskService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    endpointIdOrEndpoint: string | ExtendedEndpoint,
+    metadata: BackgroundTaskMetadata
+  ) {
+    const endpoint =
+      typeof endpointIdOrEndpoint === "string"
+        ? await findEndpoint(endpointIdOrEndpoint)
+        : endpointIdOrEndpoint;
+
+    return this.#upsertBackgroundTask(endpoint, endpoint.environment, metadata);
+  }
+
+  async #upsertBackgroundTask(
+    endpoint: Endpoint,
+    environment: AuthenticatedEnvironment,
+    metadata: BackgroundTaskMetadata
+  ): Promise<BackgroundTaskVersion | undefined> {
+    // Check the background task doesn't already exist and is deleted
+    const existingBackgroundTask = await this.#prismaClient.backgroundTask.findUnique({
+      where: {
+        projectId_slug: {
+          projectId: environment.projectId,
+          slug: metadata.id,
+        },
+      },
+    });
+
+    if (existingBackgroundTask && existingBackgroundTask.deletedAt && !metadata.enabled) {
+      return;
+    }
+
+    const backgroundTask = await this.#prismaClient.backgroundTask.upsert({
+      where: {
+        projectId_slug: {
+          projectId: environment.projectId,
+          slug: metadata.id,
+        },
+      },
+      create: {
+        organization: {
+          connect: {
+            id: environment.organizationId,
+          },
+        },
+        project: {
+          connect: {
+            id: environment.projectId,
+          },
+        },
+        slug: metadata.id,
+        title: metadata.name,
+      },
+      update: {
+        title: metadata.name,
+        deletedAt: metadata.enabled ? null : undefined,
+      },
+    });
+
+    const backgroundTaskVersion = await this.#prismaClient.backgroundTaskVersion.upsert({
+      where: {
+        backgroundTaskId_version_environmentId: {
+          backgroundTaskId: backgroundTask.id,
+          version: metadata.version,
+          environmentId: environment.id,
+        },
+      },
+      create: {
+        backgroundTask: {
+          connect: {
+            id: backgroundTask.id,
+          },
+        },
+        endpoint: {
+          connect: {
+            id: endpoint.id,
+          },
+        },
+        environment: {
+          connect: {
+            id: environment.id,
+          },
+        },
+        organization: {
+          connect: {
+            id: environment.organizationId,
+          },
+        },
+        project: {
+          connect: {
+            id: environment.projectId,
+          },
+        },
+        version: metadata.version,
+        cpu: metadata.cpu,
+        memory: metadata.memory,
+        concurrency: metadata.concurrency ?? DEFAULT_MAX_CONCURRENT_RUNS,
+        region: metadata.region ?? backgroundTaskProvider.defaultRegion,
+        diskSize: metadata.diskSizeInGB,
+      },
+      update: {
+        cpu: metadata.cpu,
+        memory: metadata.memory,
+        concurrency: metadata.concurrency ?? DEFAULT_MAX_CONCURRENT_RUNS,
+        region: metadata.region,
+        diskSize: metadata.diskSizeInGB,
+        endpoint: {
+          connect: {
+            id: endpoint.id,
+          },
+        },
+      },
+    });
+
+    // Count the number of job instances that have higher version numbers
+    const laterVersionCount = await this.#prismaClient.backgroundTaskVersion.count({
+      where: {
+        backgroundTaskId: backgroundTask.id,
+        version: {
+          gt: metadata.version,
+        },
+        environmentId: environment.id,
+      },
+    });
+
+    // If there are no later versions, then we can upsert the latest BackgroundTaskAlias
+    if (laterVersionCount === 0) {
+      await this.#prismaClient.backgroundTaskAlias.upsert({
+        where: {
+          backgroundTaskId_environmentId_name: {
+            backgroundTaskId: backgroundTask.id,
+            environmentId: environment.id,
+            name: "latest",
+          },
+        },
+        create: {
+          backgroundTaskId: backgroundTask.id,
+          versionId: backgroundTaskVersion.id,
+          environmentId: environment.id,
+          name: "latest",
+          value: backgroundTaskVersion.version,
+        },
+        update: {
+          versionId: backgroundTaskVersion.id,
+          value: backgroundTaskVersion.version,
+        },
+      });
+    }
+
+    // Now we need to register the background task secrets
+    // 1. Add new secrets
+    // 2. Remove old secrets
+    // 3. Update existing secrets
+
+    const existingSecrets = await this.#prismaClient.backgroundTaskSecret.findMany({
+      where: {
+        backgroundTaskVersionId: backgroundTaskVersion.id,
+      },
+      select: {
+        id: true,
+        key: true,
+      },
+    });
+
+    const metadataSecrets = metadata.secrets ?? {};
+
+    const existingSecretKeys = existingSecrets.map((s) => s.key);
+    const newSecretKeys = Object.keys(metadataSecrets);
+
+    const secretsToRemove = existingSecrets.filter((s) => !newSecretKeys.includes(s.key));
+    const secretsToCreate = newSecretKeys.filter((k) => !existingSecretKeys.includes(k));
+    const secretsToUpdate = newSecretKeys.filter((k) => existingSecretKeys.includes(k));
+
+    // 1. Add new secrets
+    for (const secretKey of secretsToCreate) {
+      await createBackgroundTaskSecret(
+        this.#prismaClient,
+        backgroundTaskVersion,
+        secretKey,
+        metadataSecrets[secretKey]
+      );
+    }
+
+    // 2. Remove old secrets
+    for (const secret of secretsToRemove) {
+      await deleteBackgroundTaskSecret(this.#prismaClient, secret.id);
+    }
+
+    // 3. Update existing secrets
+    for (const secretKey of secretsToUpdate) {
+      await updateBackgroundTaskSecret(
+        this.#prismaClient,
+        backgroundTaskVersion,
+        secretKey,
+        metadataSecrets[secretKey]
+      );
+    }
+
+    return backgroundTaskVersion;
+  }
+}

apps/webapp/app/services/docker/registryProxy.server.ts

@@ -0,0 +1,146 @@
+import { env } from "~/env.server";
+import { authenticateApiKey } from "../apiAuth.server";
+import { logger } from "../logger.server";
+
+export class RegistryProxy {
+  constructor(public readonly host: string, private auth: { username: string; password: string }) {}
+
+  public async call(request: Request) {
+    return await this.#proxyRequest(request);
+  }
+
+  // Proxies the request to the registry
+  async #proxyRequest(request: Request) {
+    const credentials = this.#getBasicAuthCredentials(request);
+
+    if (!credentials) {
+      logger.debug("Returning 401 because credentials are missing");
+
+      return new Response("Unauthorized", {
+        status: 401,
+        headers: {
+          "WWW-Authenticate": 'Basic realm="Access to the registry"',
+        },
+      });
+    }
+
+    // Authenticate the request
+    const authenticatedEnv = await authenticateApiKey(credentials.password, {
+      allowPublicKey: false,
+    });
+
+    if (!authenticatedEnv) {
+      return new Response("Unauthorized", {
+        status: 401,
+      });
+    }
+
+    // construct a new url based on the url passed in and the registry url
+    const proxiedUrl = new URL(request.url);
+    proxiedUrl.host = this.host;
+
+    // Update the protocol to https if there is the x-forwarded-proto header
+    if (request.headers.get("x-forwarded-proto") === "https") {
+      proxiedUrl.protocol = "https:";
+    }
+
+    const updatedHeaders = this.#updateHeaders(request.headers);
+
+    const response = await fetch(proxiedUrl, {
+      method: request.method,
+      headers: updatedHeaders,
+      body: request.body,
+    });
+
+    const updatedResponseHeaders = this.#updateResponseHeaders(response.headers, request.url);
+
+    logger.debug("proxied request/response", {
+      proxiedUrl,
+      status: response.status,
+      statusText: response.statusText,
+      method: request.method,
+      requestHeaders: Object.fromEntries(updatedHeaders.entries()),
+      responseHeaders: Object.fromEntries(updatedResponseHeaders.entries()),
+    });
+
+    return new Response(response.body, {
+      status: response.status,
+      statusText: response.statusText,
+      headers: updatedResponseHeaders,
+    });
+  }
+
+  #getBasicAuthCredentials(request: Request) {
+    const authHeader = request.headers.get("authorization");
+
+    if (!authHeader) {
+      return;
+    }
+
+    const [type, credentials] = authHeader.split(" ");
+
+    if (type.toLowerCase() !== "basic") {
+      return;
+    }
+
+    const decoded = Buffer.from(credentials, "base64").toString("utf-8");
+    const [username, password] = decoded.split(":");
+
+    return { username, password };
+  }
+
+  // Updates the headers to be sent to the registry
+  // adds the docker auth
+  #updateHeaders(headers: Headers): Headers {
+    const newHeaders = new Headers(headers);
+
+    // Remove host, connection, accept-encoding, content-length, and authorization headers
+    newHeaders.delete("host");
+    newHeaders.delete("connection");
+    newHeaders.delete("accept-encoding");
+    newHeaders.delete("authorization");
+    newHeaders.delete("content-length");
+
+    newHeaders.set(
+      "authorization",
+      `Basic ${Buffer.from(`${this.auth.username}:${this.auth.password}`).toString("base64")}`
+    );
+
+    return newHeaders;
+  }
+
+  // Updates the headers to be sent back to the client
+  #updateResponseHeaders(headers: Headers, proxyUrl: string): Headers {
+    const newHeaders = new Headers(headers);
+
+    // Rewrite location headers to point to the proxy
+    if (headers.has("location")) {
+      const location = headers.get("location");
+
+      if (location) {
+        const proxiedLocation = new URL(location);
+        proxiedLocation.host = new URL(proxyUrl).host;
+
+        newHeaders.set("location", proxiedLocation.href);
+      }
+    }
+
+    return newHeaders;
+  }
+}
+
+export async function proxyToRegistry(request: Request) {
+  if (!env.DOCKER_REGISTRY_HOST || !env.DOCKER_REGISTRY_USERNAME || !env.DOCKER_REGISTRY_PASSWORD) {
+    return new Response(
+      "Could not proxy to the registry, please double check your DOCKER_REGISTRY_* env vars",
+      { status: 500 }
+    );
+  }
+
+  const registryProxy = new RegistryProxy(env.DOCKER_REGISTRY_HOST, {
+    username: env.DOCKER_REGISTRY_USERNAME,
+    password: env.DOCKER_REGISTRY_PASSWORD,
+  });
+
+  return await registryProxy.call(request);
+}

apps/webapp/app/services/endpointApi.server.ts

@@ -1,6 +1,7 @@
 import {
   ApiEventLog,
   DeliverEventResponseSchema,
+  DeserializedJson,
   ErrorWithStackSchema,
   HttpSourceRequest,
   HttpSourceResponseSchema,
@@ -9,8 +10,8 @@ import {
   PongResponseSchema,
   PreprocessRunBody,
   PreprocessRunResponseSchema,
-  RegisterTriggerBody,
-  RegisterTriggerBodySchema,
+  RegisterTriggerBodySchemaV1,
+  RegisterTriggerBodyV1,
   RunJobBody,
   RunJobResponseSchema,
   ValidateResponse,
@@ -18,6 +19,8 @@ import {
 } from "@trigger.dev/core";
 import { safeBodyFromResponse, safeParseBodyFromResponse } from "~/utils/json";
 import { logger } from "./logger.server";
+import { ConnectionAuth } from "@trigger.dev/core";
+import { performance } from "node:perf_hooks";
 
 export class EndpointApiError extends Error {
   constructor(message: string, stack?: string) {
@@ -162,6 +165,8 @@ export class EndpointApi {
   }
 
   async executeJobRequest(options: RunJobBody) {
+    const startTimeInMs = performance.now();
+
     const response = await safeFetch(this.url, {
       method: "POST",
       headers: {
@@ -176,6 +181,7 @@ export class EndpointApi {
       response,
       parser: RunJobResponseSchema,
       errorParser: ErrorWithStackSchema,
+      durationInMs: Math.floor(performance.now() - startTimeInMs),
     };
   }
 
@@ -193,7 +199,7 @@ export class EndpointApi {
     return { response, parser: PreprocessRunResponseSchema };
   }
 
-  async initializeTrigger(id: string, params: any): Promise<RegisterTriggerBody | undefined> {
+  async initializeTrigger(id: string, params: any): Promise<RegisterTriggerBodyV1 | undefined> {
     const response = await safeFetch(this.url, {
       method: "POST",
       headers: {
@@ -227,7 +233,7 @@ export class EndpointApi {
       body: anyBody,
     });
 
-    return RegisterTriggerBodySchema.parse(anyBody);
+    return RegisterTriggerBodySchemaV1.parse(anyBody);
   }
 
   async deliverHttpSourceRequest(options: {
@@ -237,6 +243,8 @@ export class EndpointApi {
     params: any;
     data: any;
     request: HttpSourceRequest;
+    auth?: ConnectionAuth;
+    metadata?: any;
   }) {
     const response = await safeFetch(this.url, {
       method: "POST",
@@ -251,7 +259,9 @@ export class EndpointApi {
         "x-ts-http-url": options.request.url,
         "x-ts-http-method": options.request.method,
         "x-ts-http-headers": JSON.stringify(options.request.headers),
+        ...(options.auth && { "x-ts-auth": JSON.stringify(options.auth) }),
         ...(options.dynamicId && { "x-ts-dynamic-id": options.dynamicId }),
+        ...(options.metadata && { "x-ts-metadata": JSON.stringify(options.metadata) }),
       },
       body: options.request.rawBody,
     });

apps/webapp/app/services/endpoints/createEndpoint.server.ts

@@ -4,6 +4,7 @@ import { AuthenticatedEnvironment } from "../apiAuth.server";
 import { EndpointApi } from "../endpointApi.server";
 import { workerQueue } from "../worker.server";
 import { env } from "~/env.server";
+import { RuntimeEnvironmentType } from "@trigger.dev/database";
 
 const indexingHookIdentifier = customAlphabet("0123456789abcdefghijklmnopqrstuvxyz", 10);
 
@@ -51,6 +52,9 @@ export class CreateEndpointService {
               slug: id,
             },
           },
+          include: {
+            environment: true,
+          },
           create: {
             environment: {
               connect: {
@@ -83,7 +87,11 @@ export class CreateEndpointService {
             id: endpoint.id,
             source: "INTERNAL",
           },
-          { tx }
+          {
+            tx,
+            maxAttempts:
+              endpoint.environment.type === RuntimeEnvironmentType.DEVELOPMENT ? 1 : undefined,
+          }
         );
 
         return endpoint;

apps/webapp/app/services/endpoints/indexEndpoint.server.ts

@@ -4,14 +4,22 @@ import { findEndpoint } from "~/models/endpoint.server";
 import { EndpointApi } from "../endpointApi.server";
 import { RegisterJobService } from "../jobs/registerJob.server";
 import { logger } from "../logger.server";
-import { RegisterSourceService } from "../sources/registerSource.server";
+import { RegisterSourceServiceV1 } from "../sources/registerSourceV1.server";
 import { RegisterDynamicScheduleService } from "../triggers/registerDynamicSchedule.server";
 import { RegisterDynamicTriggerService } from "../triggers/registerDynamicTrigger.server";
+import { DisableJobService } from "../jobs/disableJob.server";
+import { RegisterSourceServiceV2 } from "../sources/registerSourceV2.server";
+import { RegisterBackgroundTaskService } from "../backgroundTasks/registerBackgroundTask.server";
+import { DisableBackgroundTaskService } from "../backgroundTasks/disableBackgroundTask.server";
 
 export class IndexEndpointService {
   #prismaClient: PrismaClient;
   #registerJobService = new RegisterJobService();
-  #registerSourceService = new RegisterSourceService();
+  #disableJobService = new DisableJobService();
+  #registerSourceServiceV1 = new RegisterSourceServiceV1();
+  #registerSourceServiceV2 = new RegisterSourceServiceV2();
+  #registerBackgroundTaskService = new RegisterBackgroundTaskService();
+  #disableBackgroundTaskService = new DisableBackgroundTaskService();
   #registerDynamicTriggerService = new RegisterDynamicTriggerService();
   #registerDynamicScheduleService = new RegisterDynamicScheduleService();
 
@@ -36,7 +44,13 @@ export class IndexEndpointService {
       throw new Error(indexResponse.error);
     }
 
-    const { jobs, sources, dynamicTriggers, dynamicSchedules } = indexResponse.data;
+    const {
+      jobs,
+      sources,
+      dynamicTriggers,
+      dynamicSchedules,
+      backgroundTasks = [],
+    } = indexResponse.data;
 
     logger.debug("Indexing endpoint", {
       endpointId: endpoint.id,
@@ -49,37 +63,219 @@ export class IndexEndpointService {
         sources: sources.length,
         dynamicTriggers: dynamicTriggers.length,
         dynamicSchedules: dynamicSchedules.length,
+        backgroundTasks: backgroundTasks.length,
       },
     });
 
     const indexStats = {
       jobs: 0,
+      backgroundTasks: 0,
       sources: 0,
       dynamicTriggers: 0,
       dynamicSchedules: 0,
+      disabledJobs: 0,
+      disabedBackgroundTasks: 0,
     };
 
+    const existingJobs = await this.#prismaClient.job.findMany({
+      where: {
+        projectId: endpoint.projectId,
+        deletedAt: null,
+      },
+      include: {
+        aliases: {
+          where: {
+            name: "latest",
+            environmentId: endpoint.environmentId,
+          },
+          include: {
+            version: true,
+          },
+          take: 1,
+        },
+      },
+    });
+
     for (const job of jobs) {
       if (!job.enabled) {
-        continue;
+        const disabledJob = await this.#disableJobService
+          .call(endpoint, { slug: job.id, version: job.version })
+          .catch((error) => {
+            logger.error("Failed to disable job", {
+              endpointId: endpoint.id,
+              job,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledJob) {
+          indexStats.disabledJobs++;
+        }
+      } else {
+        try {
+          const registeredVersion = await this.#registerJobService.call(endpoint, job);
+
+          if (registeredVersion) {
+            indexStats.jobs++;
+          }
+        } catch (error) {
+          logger.error("Failed to register job", {
+            endpointId: endpoint.id,
+            job,
+            error,
+          });
+        }
       }
+    }
 
-      try {
-        await this.#registerJobService.call(endpoint, job);
+    // TODO: we need to do this for sources, dynamic triggers, and dynamic schedules
+    const missingJobs = existingJobs.filter((job) => {
+      return !jobs.find((j) => j.id === job.slug);
+    });
 
-        indexStats.jobs++;
-      } catch (error) {
-        logger.error("Failed to register job", {
-          endpointId: endpoint.id,
-          job,
-          error,
-        });
+    if (missingJobs.length > 0) {
+      logger.debug("Disabling missing jobs", {
+        endpointId: endpoint.id,
+        missingJobIds: missingJobs.map((job) => job.slug),
+      });
+
+      for (const job of missingJobs) {
+        const latestVersion = job.aliases[0]?.version;
+
+        if (!latestVersion) {
+          continue;
+        }
+
+        const disabledJob = await this.#disableJobService
+          .call(endpoint, {
+            slug: job.slug,
+            version: latestVersion.version,
+          })
+          .catch((error) => {
+            logger.error("Failed to disable job", {
+              endpointId: endpoint.id,
+              job,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledJob) {
+          indexStats.disabledJobs++;
+        }
+      }
+    }
+
+    const existingBackgroundTasks = await this.#prismaClient.backgroundTask.findMany({
+      where: {
+        projectId: endpoint.projectId,
+        deletedAt: null,
+      },
+      include: {
+        aliases: {
+          where: {
+            name: "latest",
+            environmentId: endpoint.environmentId,
+          },
+          include: {
+            version: true,
+          },
+          take: 1,
+        },
+      },
+    });
+
+    for (const backgroundTask of backgroundTasks) {
+      if (!backgroundTask.enabled) {
+        const disabledBackgroundTask = await this.#disableBackgroundTaskService
+          .call(endpoint, { slug: backgroundTask.id, version: backgroundTask.version })
+          .catch((error) => {
+            logger.error("Failed to disable background task", {
+              endpointId: endpoint.id,
+              backgroundTask,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledBackgroundTask) {
+          indexStats.disabledJobs++;
+        }
+      } else {
+        try {
+          const registeredVersion = await this.#registerBackgroundTaskService.call(
+            endpoint,
+            backgroundTask
+          );
+
+          if (registeredVersion) {
+            indexStats.backgroundTasks++;
+          }
+        } catch (error) {
+          logger.error("Failed to register background task", {
+            endpointId: endpoint.id,
+            backgroundTask,
+            error,
+          });
+        }
+      }
+    }
+
+    const missingBackgroundTasks = existingBackgroundTasks.filter((backgroundTask) => {
+      return !backgroundTasks.find((b) => b.id === backgroundTask.slug);
+    });
+
+    if (missingBackgroundTasks.length > 0) {
+      logger.debug("Disabling missing background tasks", {
+        endpointId: endpoint.id,
+        missingIds: missingBackgroundTasks.map((job) => job.slug),
+      });
+
+      for (const backgroundTask of missingBackgroundTasks) {
+        const latestVersion = backgroundTask.aliases[0]?.version;
+
+        if (!latestVersion) {
+          continue;
+        }
+
+        const disabledBackgroundTask = await this.#disableBackgroundTaskService
+          .call(endpoint, {
+            slug: backgroundTask.slug,
+            version: latestVersion.version,
+          })
+          .catch((error) => {
+            logger.error("Failed to disable background task", {
+              endpointId: endpoint.id,
+              backgroundTask,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledBackgroundTask) {
+          indexStats.disabledJobs++;
+        }
       }
     }
 
     for (const source of sources) {
       try {
-        await this.#registerSourceService.call(endpoint, source);
+        switch (source.version) {
+          default:
+          case "1": {
+            await this.#registerSourceServiceV1.call(endpoint, source);
+            break;
+          }
+          case "2": {
+            await this.#registerSourceServiceV2.call(endpoint, source);
+            break;
+          }
+        }
 
         indexStats.sources++;
       } catch (error) {

apps/webapp/app/services/endpoints/validateCreateEndpoint.server.ts

@@ -5,6 +5,7 @@ import { AuthenticatedEnvironment } from "../apiAuth.server";
 import { workerQueue } from "../worker.server";
 import { CreateEndpointError } from "./createEndpoint.server";
 import { EndpointApi } from "../endpointApi.server";
+import { RuntimeEnvironmentType } from "@trigger.dev/database";
 
 const indexingHookIdentifier = customAlphabet("0123456789abcdefghijklmnopqrstuvxyz", 10);
 
@@ -35,6 +36,9 @@ export class ValidateCreateEndpointService {
               slug: validationResult.endpointId,
             },
           },
+          include: {
+            environment: true,
+          },
           create: {
             environment: {
               connect: {
@@ -67,7 +71,11 @@ export class ValidateCreateEndpointService {
             id: endpoint.id,
             source: "INTERNAL",
           },
-          { tx }
+          {
+            tx,
+            maxAttempts:
+              endpoint.environment.type === RuntimeEnvironmentType.DEVELOPMENT ? 1 : undefined,
+          }
         );
 
         return endpoint;

apps/webapp/app/services/events/cancelEvent.server.ts

@@ -0,0 +1,47 @@
+import type { EventRecord } from "@trigger.dev/database";
+import { $transaction, PrismaClientOrTransaction, prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+
+export class CancelEventService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    environment: AuthenticatedEnvironment,
+    eventId: string
+  ): Promise<EventRecord | undefined> {
+    return await $transaction(this.#prismaClient, async (tx) => {
+      const event = await tx.eventRecord.findUnique({
+        where: {
+          eventId_environmentId: {
+            eventId: eventId,
+            environmentId: environment.id,
+          },
+        },
+      });
+
+      if (!event) {
+        return;
+      }
+
+      if (event.cancelledAt) {
+        return event;
+      }
+
+      //update the cancelledAt column in the eventRecord table
+      const updatedEvent = await tx.eventRecord.update({
+        where: { id: event.id },
+        data: { cancelledAt: new Date() },
+      });
+
+      // Dequeue the event after the db has been updated
+      await workerQueue.dequeue(`event:${event.id}`, { tx });
+
+      return updatedEvent;
+    });
+  }
+}

apps/webapp/app/services/events/ingestSendEvent.server.ts

@@ -2,6 +2,7 @@ import type { RawEvent, SendEventOptions } from "@trigger.dev/core";
 import { $transaction, PrismaClientOrTransaction, PrismaErrorSchema, prisma } from "~/db.server";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { workerQueue } from "~/services/worker.server";
+import { logger } from "../logger.server";
 
 export class IngestSendEvent {
   #prismaClient: PrismaClientOrTransaction;
@@ -36,71 +37,87 @@ export class IngestSendEvent {
     try {
       const deliverAt = this.#calculateDeliverAt(options);
 
-      return await $transaction(this.#prismaClient, async (tx) => {
-        const externalAccount = options?.accountId
-          ? await tx.externalAccount.findUniqueOrThrow({
-              where: {
-                environmentId_identifier: {
-                  environmentId: environment.id,
-                  identifier: options.accountId,
+      return await $transaction(
+        this.#prismaClient,
+        async (tx) => {
+          const externalAccount = options?.accountId
+            ? await tx.externalAccount.findUniqueOrThrow({
+                where: {
+                  environmentId_identifier: {
+                    environmentId: environment.id,
+                    identifier: options.accountId,
+                  },
+                },
+              })
+            : undefined;
+
+          // Create a new event in the database
+          const eventLog = await tx.eventRecord.create({
+            data: {
+              organization: {
+                connect: {
+                  id: environment.organizationId,
                 },
               },
-            })
-          : undefined;
-
-        // Create a new event in the database
-        const eventLog = await tx.eventRecord.create({
-          data: {
-            organization: {
-              connect: {
-                id: environment.organizationId,
+              project: {
+                connect: {
+                  id: environment.projectId,
+                },
               },
-            },
-            project: {
-              connect: {
-                id: environment.projectId,
+              environment: {
+                connect: {
+                  id: environment.id,
+                },
               },
+              eventId: event.id,
+              name: event.name,
+              timestamp: event.timestamp ?? new Date(),
+              payload: event.payload ?? {},
+              context: event.context ?? {},
+              source: event.source ?? "trigger.dev",
+              sourceContext,
+              deliverAt: deliverAt,
+              externalAccount: externalAccount
+                ? {
+                    connect: {
+                      id: externalAccount.id,
+                    },
+                  }
+                : {},
             },
-            environment: {
-              connect: {
-                id: environment.id,
+          });
+
+          if (this.deliverEvents) {
+            // Produce a message to the event bus
+            await workerQueue.enqueue(
+              "deliverEvent",
+              {
+                id: eventLog.id,
               },
-            },
-            eventId: event.id,
-            name: event.name,
-            timestamp: event.timestamp ?? new Date(),
-            payload: event.payload ?? {},
-            context: event.context ?? {},
-            source: event.source ?? "trigger.dev",
-            sourceContext,
-            deliverAt: deliverAt,
-            externalAccount: externalAccount
-              ? {
-                  connect: {
-                    id: externalAccount.id,
-                  },
-                }
-              : {},
-          },
-        });
+              { runAt: eventLog.deliverAt, tx, jobKey: `event:${eventLog.id}` }
+            );
+          }
 
-        if (this.deliverEvents) {
-          // Produce a message to the event bus
-          await workerQueue.enqueue(
-            "deliverEvent",
-            {
-              id: eventLog.id,
-            },
-            { runAt: eventLog.deliverAt, tx }
-          );
-        }
-
-        return eventLog;
-      });
+          return eventLog;
+        },
+        { rethrowPrismaErrors: true }
+      );
     } catch (error) {
       const prismaError = PrismaErrorSchema.safeParse(error);
+
+      if (!prismaError.success) {
+        logger.debug("Error parsing prisma error", {
+          error,
+          parseError: prismaError.error.format(),
+        });
+
+        throw error;
+      }
+
       // If the error is a Prisma unique constraint error, it means that the event already exists
       if (prismaError.success && prismaError.data.code === "P2002") {
+        logger.debug("Event already exists, finding and returning", { event, environment });
+
         return this.#prismaClient.eventRecord.findUniqueOrThrow({
           where: {
             eventId_environmentId: {