Update pnpm lock

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/e2e.yml

@@ -0,0 +1,65 @@
+name: "🧪 E2E Tests"
+on:
+  workflow_call:
+jobs:
+  e2e:
+    name: "🧪 E2E Tests"
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    steps:
+      - name: 🐳 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 7.18
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: |
+          # Setup environment variables
+          cp ./.env.example ./.env
+          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
+
+          # Build packages
+          pnpm run build --filter @references/nextjs-test^...
+          pnpm --filter @trigger.dev/database generate
+
+          # Move trigger-cli bin to correct place
+          pnpm install --frozen-lockfile
+
+          # Execute tests
+          pnpm run docker
+          pnpm run db:migrate
+          pnpm run db:seed
+          pnpm run test:e2e
+
+          # Cleanup
+          pnpm run docker:stop
+
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.github/workflows/pr_checks.yml

@@ -0,0 +1,31 @@
+name: 🤖 PR Checks
+
+on:
+  pull_request_target:
+    branches:
+      - main
+    paths-ignore:
+      - "**.md"
+      - ".github/CODEOWNERS"
+      - ".github/ISSUE_TEMPLATE/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  typecheck:
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
+
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
+
+  e2e:
+    uses: ./.github/workflows/e2e.yml
+    secrets: inherit

.github/workflows/publish.yml

@@ -38,125 +38,19 @@ env:
 
 jobs:
   typecheck:
-    name: ʦ TypeScript
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
 
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: 📀 Generate Prisma Client
-        run: pnpm run generate
-
-      - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
-
-  unitTests:
-    name: Unit Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Run Unit Tests
-        run: |
-          pnpm run test
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
 
   e2e:
-    name: e2e Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
-
-      - name: Run Playwright tests
-        run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
-
-          # Build packages
-          pnpm run build --filter @references/nextjs-test^...
-          pnpm --filter @trigger.dev/database generate
-
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+    uses: ./.github/workflows/e2e.yml
+    secrets: inherit
 
   publish:
-    needs: [typecheck, unitTests, e2e]
+    needs: [typecheck, units, e2e]
     runs-on: buildjet-4vcpu-ubuntu-2204
     outputs:
       version: ${{ steps.get_version.outputs.version }}

.github/workflows/release.yml

@@ -4,16 +4,11 @@ on:
   push:
     branches:
       - main
-    paths:
-      - ".github/workflows/release.yml"
-      - "packages/**"
-      - "!packages/**/*.md"
-      - ".changeset/**"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "pnpm-lock.yaml"
-      - "pnpm-workspace.yaml"
-      - "turbo.json"
+    paths-ignore:
+      - "**.md"
+      - ".github/CODEOWNERS"
+      - ".github/ISSUE_TEMPLATE/**"
+    
 
 jobs:
   release:

.github/workflows/typecheck.yml

@@ -0,0 +1,32 @@
+name: "ʦ TypeScript"
+on:
+  workflow_call:
+jobs:
+  typecheck:
+    runs-on: buildjet-4vcpu-ubuntu-2204
+
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 7.18
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔎 Type check
+        run: pnpm run typecheck --filter webapp

.github/workflows/unit-tests.yml

@@ -0,0 +1,30 @@
+name: "🧪 Unit Tests"
+on:
+  workflow_call:
+jobs:
+  unitTests:
+    name: "🧪 Unit Tests"
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 7.18
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Run Unit Tests
+        run: |
+          pnpm run test

.nvmrc

@@ -1 +1 @@
-v18.12.1
+v18.18.0

.vscode/extensions.json

@@ -0,0 +1,9 @@
+{
+	"recommendations": [
+        "astro-build.astro-vscode",
+        "denoland.vscode-deno"
+    ],
+	"unwantedRecommendations": [
+		
+	]
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "deno.enablePaths": ["references/deno-reference"]
+}

CHANGESETS.md

@@ -32,7 +32,26 @@ Please follow the best-practice of adding changesets in the same commit as the c
 
 !MAKE SURE TO UPDATE THE TAG IN THE INSTRUCTIONS BELOW!
 
-1. Add changesets as usual `pnpm run changeset:add`
-2. Create a snapshot version (replace "dev" with your tag) `pnpm exec changeset version --snapshot dev`
-3. Build the packages: `pnpm run build --filter "@trigger.dev/*"`
-4. Publish the snapshot (replace "dev" with your tag) `pnpm exec changeset publish --no-git-tag --snapshot --tag dev`
+1. Add changesets as usual
+
+```sh
+pnpm run changeset:add
+```
+
+2. Create a snapshot version (replace "prerelease" with your tag)
+
+```sh
+pnpm exec changeset version --snapshot prerelease
+```
+
+3. Build the packages:
+
+```sh
+pnpm run build --filter "@trigger.dev/*"
+```
+
+4. Publish the snapshot (replace "dev" with your tag)
+
+```sh
+pnpm exec changeset publish --no-git-tag --snapshot --tag prerelease
+```

CONTRIBUTING.md

@@ -52,9 +52,13 @@ branch are tagged into a release monthly.
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
 6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+
    ```
    pnpm run docker
    ```
+
+   This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
+
 7. Migrate the database
    ```
    pnpm run db:migrate

README.md

@@ -14,6 +14,18 @@
 
 </div>
 
+# ✨🎃 Get involved with Hacktoberfest 2023! 🎃✨
+
+All of October we're participating in Hacktoberfest and invite you to join us! We have a bunch of issues labeled `🎃 Hacktoberfest` that are ready for you to work on which will count towards Hacktoberfest. We are also running our own game, earn 💎 points to win swag!
+
+- Check out our [Hacktoberfest landing page](https://trigger.dev/hacktoberfest) for how to participate and win swag.
+- Contribute to either our [/trigger.dev](https://github.com/triggerdotdev/trigger.dev/labels/%F0%9F%8E%83%20hacktoberfest), [/api-reference](https://github.com/triggerdotdev/api-reference/issues?q=is%3Aopen+is%3Aissue+label%3A%F0%9F%8E%83hacktoberfest) or [/jobs-showcase](https://github.com/triggerdotdev/jobs-showcase/labels/%F0%9F%8E%83%20hacktoberfest) repositories and complete issues marked `🎃 Hacktoberfest` to be eligible for swag.
+- Join our [Discord](https://discord.gg/JtBAxBr2m3) and get involved in with the community.
+
+_New to Hacktober? Check out the [Hacktoberfest website](https://hacktoberfest.digitalocean.com/) for more information._
+
+🎃 **Happy Hacking!** 🎃
+
 # About Trigger.dev
 
 Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
@@ -62,8 +74,8 @@ Click the links to join the discussions about our upcoming features.
 | Dashboard                                                                            | View every Task in every Run                        | ✅     |
 | Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
 | React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
+| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
 | [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [React frameworks](https://github.com/triggerdotdev/trigger.dev/discussions/411)     | Support for Remix, Astro, RedwoodJS & more          | 🛠️     |
 | [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
 | Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
 | Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
@@ -83,3 +95,9 @@ We provide an official trigger.dev docker image you can use to easily self-host
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
+
+## Meet the Amazing People Behind This Project 🚀
+
+<a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />
+</a>

apps/webapp/app/components/SetupCommands.tsx

@@ -44,28 +44,7 @@ export function InitCommand({ appOrigin, apiKey }: { appOrigin: string; apiKey:
   );
 }
 
-export function RunDevCommand() {
-  return (
-    <ClientTabs defaultValue="npm">
-      <ClientTabsList>
-        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
-        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
-        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
-      </ClientTabsList>
-      <ClientTabsContent value={"npm"}>
-        <ClipboardField variant="primary/medium" className="mb-4" value={`npm run dev`} />
-      </ClientTabsContent>
-      <ClientTabsContent value={"pnpm"}>
-        <ClipboardField variant="primary/medium" className="mb-4" value={`pnpm run dev`} />
-      </ClientTabsContent>
-      <ClientTabsContent value={"yarn"}>
-        <ClipboardField variant="primary/medium" className="mb-4" value={`yarn run dev`} />
-      </ClientTabsContent>
-    </ClientTabs>
-  );
-}
-
-export function TriggerDevCommand() {
+export function RunDevCommand({ extra }: { extra?: string }) {
   return (
     <ClientTabs defaultValue="npm">
       <ClientTabsList>
@@ -77,34 +56,67 @@ export function TriggerDevCommand() {
         <ClipboardField
           variant="primary/medium"
           className="mb-4"
-          value={`npx @trigger.dev/cli@latest dev`}
+          value={`npm run dev${extra ? ` ${extra}` : ""}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"pnpm"}>
         <ClipboardField
           variant="primary/medium"
           className="mb-4"
-          value={`pnpm dlx @trigger.dev/cli@latest dev`}
+          value={`pnpm run dev${extra ? ` ${extra}` : ""}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"yarn"}>
         <ClipboardField
           variant="primary/medium"
           className="mb-4"
-          value={`yarn dlx @trigger.dev/cli@latest dev`}
+          value={`yarn run dev${extra ? ` ${extra}` : ""}`}
         />
       </ClientTabsContent>
     </ClientTabs>
   );
 }
 
-export function TriggerDevStep() {
+export function TriggerDevCommand({ extra }: { extra?: string }) {
+  return (
+    <ClientTabs defaultValue="npm">
+      <ClientTabsList>
+        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
+      </ClientTabsList>
+      <ClientTabsContent value={"npm"}>
+        <ClipboardField
+          variant="primary/medium"
+          className="mb-4"
+          value={`npx @trigger.dev/cli@latest dev${extra ? ` ${extra}` : ""}`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"pnpm"}>
+        <ClipboardField
+          variant="primary/medium"
+          className="mb-4"
+          value={`pnpm dlx @trigger.dev/cli@latest dev${extra ? ` ${extra}` : ""}`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"yarn"}>
+        <ClipboardField
+          variant="primary/medium"
+          className="mb-4"
+          value={`yarn dlx @trigger.dev/cli@latest dev${extra ? ` ${extra}` : ""}`}
+        />
+      </ClientTabsContent>
+    </ClientTabs>
+  );
+}
+
+export function TriggerDevStep({ extra }: { extra?: string }) {
   return (
     <>
       <Paragraph spacing>
         In a <span className="text-amber-400">separate terminal window or tab</span> run:
       </Paragraph>
-      <TriggerDevCommand />
+      <TriggerDevCommand extra={extra} />
       <Paragraph spacing variant="small">
         If you’re not running on the default you can specify the port by adding{" "}
         <InlineCode variant="extra-small">--port 3001</InlineCode> to the end.

apps/webapp/app/components/environments/EndpointIndexStatus.tsx

@@ -0,0 +1,74 @@
+import { CheckCircleIcon, ClockIcon, XCircleIcon } from "@heroicons/react/20/solid";
+import { EndpointIndexStatus } from "@trigger.dev/database";
+import { cn } from "~/utils/cn";
+import { Spinner } from "../primitives/Spinner";
+
+export function EndpointIndexStatusIcon({ status }: { status: EndpointIndexStatus }) {
+  switch (status) {
+    case "PENDING":
+      return <ClockIcon className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />;
+    case "STARTED":
+      return <Spinner className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />;
+    case "SUCCESS":
+      return (
+        <CheckCircleIcon className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />
+      );
+    case "FAILURE":
+      return <XCircleIcon className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />;
+  }
+}
+
+export function EndpointIndexStatusLabel({ status }: { status: EndpointIndexStatus }) {
+  switch (status) {
+    case "PENDING":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+    case "STARTED":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+    case "SUCCESS":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+    case "FAILURE":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+  }
+}
+
+export function endpointIndexStatusTitle(status: EndpointIndexStatus): string {
+  switch (status) {
+    case "PENDING":
+      return "Pending";
+    case "STARTED":
+      return "Started";
+    case "SUCCESS":
+      return "Success";
+    case "FAILURE":
+      return "Failure";
+  }
+}
+
+export function endpointIndexStatusClassNameColor(status: EndpointIndexStatus): string {
+  switch (status) {
+    case "PENDING":
+      return "text-dimmed";
+    case "STARTED":
+      return "text-blue-500";
+    case "SUCCESS":
+      return "text-green-500";
+    case "FAILURE":
+      return "text-rose-500";
+  }
+}

apps/webapp/app/components/frameworks/FrameworkSelector.tsx

@@ -66,7 +66,7 @@ export function FrameworkSelector() {
           <FrameworkLink to={projectSetupNuxtPath(organization, project)}>
             <NuxtLogo className="w-32" />
           </FrameworkLink>
-          <FrameworkLink to={projectSetupSvelteKitPath(organization, project)}>
+          <FrameworkLink to={projectSetupSvelteKitPath(organization, project)} supported>
             <SvelteKitLogo className="w-44" />
           </FrameworkLink>
           <FrameworkLink to={projectSetupFastifyPath(organization, project)}>

apps/webapp/app/components/navigation/NavBar.tsx

@@ -17,7 +17,6 @@ export function NavBar() {
           <LogoIcon className="h-5 w-5" />
         </Link>
         <Breadcrumb />
-        <MobileDropdownMenu />
       </div>
       <div className="hidden items-center gap-2 sm:flex">
         <LinkButton to={docsRoot()} variant="secondary/small" LeadingIcon={BookOpenIcon}>
@@ -46,83 +45,3 @@ export function BreadcrumbLink({ title, to }: { title: string; to: string }) {
     </LinkButton>
   );
 }
-
-function MobileDropdownMenu() {
-  return (
-    <Popover className="block sm:hidden">
-      <Popover.Button
-        className="bg-slate-70 relative z-10 flex h-8 w-8 items-center justify-center rounded border-none bg-opacity-50 focus-visible:border-none focus-visible:outline-none"
-        aria-label="Toggle Navigation"
-      >
-        {({ open }) => <MobileNavIcon open={open} />}
-      </Popover.Button>
-      <Transition.Root>
-        <Transition.Child
-          as={Fragment}
-          enter="duration-150 ease-out"
-          enterFrom="opacity-0"
-          enterTo="opacity-100"
-          leave="duration-150 ease-in"
-          leaveFrom="opacity-100"
-          leaveTo="opacity-0"
-        >
-          <Popover.Overlay className="absolute left-0 top-0 h-full w-full origin-top bg-midnight-900/80" />
-        </Transition.Child>
-        <Transition.Child
-          as={Fragment}
-          enter="duration-150 ease-out"
-          enterFrom="opacity-0 scale-95"
-          enterTo="opacity-100 scale-100"
-          leave="duration-100 ease-in"
-          leaveFrom="opacity-100 scale-100"
-          leaveTo="opacity-0 scale-95"
-        >
-          <Popover.Panel
-            as="div"
-            className="absolute inset-x-6 top-0 mt-20 flex origin-top flex-col gap-4 rounded-md bg-slate-800 p-4 text-lg tracking-tight text-slate-900 shadow-xl ring-1 ring-slate-700"
-          >
-            {/* <PrimaryA
-              href={docsRoot()}
-              target="_blank"
-              className="max-w-full"
-            >
-              <ArrowTopRightOnSquareIcon className="-ml-1 h-4 w-4" />
-              Documentation
-            </PrimaryA>
-            <PrimaryButton
-              data-attr="posthog-feedback-button"
-              className="max-w-full"
-            >
-              <ChatBubbleLeftRightIcon className="-ml-1 h-4 w-4" />
-              Send us feedback
-            </PrimaryButton>
-            <SecondaryLink to="/logout" className="max-w-full">
-              Logout
-            </SecondaryLink> */}
-          </Popover.Panel>
-        </Transition.Child>
-      </Transition.Root>
-    </Popover>
-  );
-}
-
-function MobileNavIcon({ open }: { open: boolean }) {
-  return (
-    <svg
-      aria-hidden="true"
-      className="h-3.5 w-3.5 overflow-visible stroke-slate-300"
-      fill="none"
-      strokeWidth={2}
-      strokeLinecap="round"
-    >
-      <path
-        d="M0 1H14M0 7H14M0 13H14"
-        className={cn("origin-center transition", open && "scale-90 opacity-0")}
-      />
-      <path
-        d="M2 2L12 12M12 2L2 12"
-        className={cn("origin-center transition", !open && "scale-90 opacity-0")}
-      />
-    </svg>
-  );
-}

apps/webapp/app/components/primitives/Callout.tsx

@@ -13,6 +13,7 @@ import {
 import { Link } from "@remix-run/react";
 import { cn } from "~/utils/cn";
 import { Paragraph } from "./Paragraph";
+import { Spinner } from "./Spinner";
 
 export const variantClasses = {
   info: {
@@ -51,8 +52,16 @@ export const variantClasses = {
     textColor: "text-blue-200",
     linkClassName: "transition hover:bg-blue-400/40",
   },
+  pending: {
+    className: "border-blue-400/20 bg-blue-800/30",
+    icon: <Spinner className="h-5 w-5 shrink-0 " />,
+    textColor: "text-blue-300",
+    linkClassName: "transition hover:bg-blue-400/40",
+  },
 } as const;
 
+export type CalloutVariant = keyof typeof variantClasses;
+
 export function Callout({
   children,
   className,
@@ -63,7 +72,7 @@ export function Callout({
   children?: React.ReactNode;
   className?: string;
   icon?: React.ReactNode;
-  variant: keyof typeof variantClasses;
+  variant: CalloutVariant;
   to?: string;
 }) {
   const variantDefinition = variantClasses[variant];

apps/webapp/app/components/primitives/FormError.tsx

@@ -2,8 +2,17 @@ import type { z } from "zod";
 import { Paragraph } from "./Paragraph";
 import { NamedIcon } from "./NamedIcon";
 import { motion } from "framer-motion";
+import { cn } from "~/utils/cn";
 
-export function FormError({ children, id }: { children: React.ReactNode; id?: string }) {
+export function FormError({
+  children,
+  id,
+  className,
+}: {
+  children: React.ReactNode;
+  id?: string;
+  className?: string;
+}) {
   return (
     <>
       {children && (
@@ -11,7 +20,7 @@ export function FormError({ children, id }: { children: React.ReactNode; id?: st
           initial={{ opacity: 0 }}
           animate={{ opacity: 1 }}
           transition={{ duration: 0.3 }}
-          className="flex items-start gap-0.5"
+          className={cn("flex items-start gap-0.5", className)}
         >
           <NamedIcon name="error" className="h-4 w-4 shrink-0 justify-start" />
           <Paragraph id={id} variant="extra-small" className="text-rose-500">

apps/webapp/app/components/primitives/NamedIcon.tsx

@@ -68,6 +68,8 @@ import { Spinner } from "./Spinner";
 import { SaplingIcon } from "~/assets/icons/SaplingIcon";
 import { TwoTreesIcon } from "~/assets/icons/TwoTreesIcon";
 import { OneTreeIcon } from "~/assets/icons/OneTreeIcon";
+import { tablerIcons } from "~/utils/tablerIcons";
+import tablerSpritePath from "./tabler-sprite.svg";
 
 const icons = {
   account: (className: string) => <UserCircleIcon className={cn("text-slate-400", className)} />,
@@ -215,7 +217,15 @@ export function NamedIcon({
     );
   }
 
-  console.log(`Icon ${name} not found`);
+  if (tablerIcons.has("tabler-" + name)) {
+    return <TablerIcon name={"tabler-" + name} className={className} />;
+  } else if (name.startsWith("tabler-") && tablerIcons.has(name)) {
+    return <TablerIcon name={name} className={className} />;
+  }
+
+  if (name === "supabase-management") {
+    return <NamedIcon name="supabase" className={className} />;
+  }
 
   if (fallback) {
     return fallback;
@@ -247,3 +257,11 @@ export function NamedIconInBox({
     </div>
   );
 }
+
+export function TablerIcon({ name, className }: { name: string; className?: string }) {
+  return (
+    <svg className={cn("stroke-[1.5]", className)}>
+      <use xlinkHref={`${tablerSpritePath}#${name}`} />
+    </svg>
+  );
+}

apps/webapp/app/components/primitives/Switch.tsx

@@ -15,7 +15,7 @@ const variations = {
     container: "flex items-center gap-x-1.5 rounded hover:bg-slate-850 pr-1 py-[0.1rem] pl-1.5",
     root: "h-3 w-6",
     thumb: "h-2.5 w-2.5 data-[state=checked]:translate-x-2.5 data-[state=unchecked]:translate-x-0",
-    text: "text-xs text-slate-400 group-hover:text-slate-200 mt-0.5",
+    text: "text-xs text-slate-400 group-hover:text-slate-200 hover:cursor-pointer",
   },
 };
 

apps/webapp/app/components/primitives/Toast.tsx

@@ -1,7 +1,7 @@
 import { ExclamationCircleIcon, XMarkIcon } from "@heroicons/react/20/solid";
 import { CheckCircleIcon } from "@heroicons/react/24/solid";
-import { AnimatePresence, motion } from "framer-motion";
-import toast, { Toaster, resolveValue, useToasterStore } from "react-hot-toast";
+import { Toaster, toast } from "sonner";
+
 import { useTypedLoaderData } from "remix-typedjson";
 import { loader } from "~/root";
 import { useEffect } from "react";
@@ -11,79 +11,55 @@ const permanentToastDuration = 60 * 60 * 24 * 1000;
 
 export function Toast() {
   const { toastMessage } = useTypedLoaderData<typeof loader>();
-
   useEffect(() => {
     if (!toastMessage) {
       return;
     }
     const { message, type, options } = toastMessage;
 
-    switch (type) {
-      case "success":
-        toast.success(message, {
-          duration: options.ephemeral ? defaultToastDuration : permanentToastDuration,
-        });
-        break;
-      case "error":
-        toast.error(message, {
-          duration: options.ephemeral ? defaultToastDuration : permanentToastDuration,
-        });
-        break;
-      default:
-        throw new Error(`${type} is not handled`);
-    }
+    toast.custom((t) => <ToastUI variant={type} message={message} t={t as string} />, {
+      duration: options.ephemeral ? defaultToastDuration : permanentToastDuration,
+    });
   }, [toastMessage]);
 
+  return <Toaster />;
+}
+
+export function ToastUI({
+  variant,
+  message,
+  t,
+  toastWidth = 356, // Default width, matches what sonner provides by default
+}: {
+  variant: "error" | "success";
+  message: string;
+  t: string;
+  toastWidth?: string | number;
+}) {
   return (
-    <Toaster
-      position="bottom-right"
-      toastOptions={{
-        success: {
-          icon: <CheckCircleIcon className="h-6 w-6 text-green-600" />,
-        },
-        error: {
-          icon: <ExclamationCircleIcon className="h-6 w-6 text-rose-600" />,
-        },
+    <div
+      className={`self-end rounded-lg border border-slate-750 bg-midnight-900 shadow-md`}
+      style={{
+        width: toastWidth,
       }}
     >
-      {(t) => (
-        <AnimatePresence>
-          <motion.div
-            className="flex gap-2 rounded-lg border border-slate-750 bg-no-repeat p-4 text-bright shadow-md"
-            style={{
-              opacity: t.visible ? 1 : 0,
-              background:
-                "radial-gradient(at top, hsla(271, 91%, 65%, 0.18), hsla(221, 83%, 53%, 0.18)) hsla(221, 83%, 53%, 0.18)",
-            }}
-            initial={{ opacity: 0, y: 100 }}
-            animate={t.visible ? "visible" : "hidden"}
-            variants={{
-              hidden: {
-                opacity: 0,
-                y: 0,
-                transition: {
-                  duration: 0.15,
-                  ease: "easeInOut",
-                },
-              },
-              visible: {
-                opacity: 1,
-                y: 0,
-                transition: {
-                  duration: 0.3,
-                  ease: "easeInOut",
-                },
-              },
-            }}
-          >
-            {t.icon}
-            {resolveValue(t.message, t)}
-            <button className="p-1" onClick={() => toast.dismiss(t.id)}>
-              <XMarkIcon className="h-4 w-4 text-bright" />
-            </button>
-          </motion.div>
-        </AnimatePresence>
-      )}
-    </Toaster>
+      <div
+        className="flex w-full gap-2 rounded-lg bg-no-repeat p-4 text-bright"
+        style={{
+          background:
+            "radial-gradient(at top, hsla(271, 91%, 65%, 0.18), hsla(221, 83%, 53%, 0.18)) hsla(221, 83%, 53%, 0.18)",
+        }}
+      >
+        {variant === "success" ? (
+          <CheckCircleIcon className="h-6 w-6 text-green-600" />
+        ) : (
+          <ExclamationCircleIcon className="h-6 w-6 text-rose-600" />
+        )}
+        {message}
+        <button className="ms-auto p-1" onClick={() => toast.dismiss(t)}>
+          <XMarkIcon className="h-4 w-4 text-bright" />
+        </button>
+      </div>
+    </div>
   );
 }

apps/webapp/app/components/stories/NamedIcon.stories.tsx

@@ -1,6 +1,8 @@
 import type { Meta, StoryObj } from "@storybook/react";
 import { withDesign } from "storybook-addon-designs";
 import { NamedIcon, iconNames } from "../primitives/NamedIcon";
+import { tablerIcons } from "~/utils/tablerIcons";
+import { Header1 } from "../primitives/Headers";
 
 const meta: Meta<typeof NamedIcons> = {
   title: "Icons",
@@ -25,17 +27,35 @@ export const Basic: Story = {
 
 function NamedIcons() {
   return (
-    <div className="grid grid-cols-8 gap-4">
-      {iconNames
-        .sort((a, b) => a.localeCompare(b))
-        .map((iconName) => (
-          <div key={iconName} className="flex items-center gap-2">
-            <div>
-              <NamedIcon name={iconName} className={"h-6 w-6"} />
+    <div className="flex flex-col gap-4">
+      <div>
+        <Header1 spacing>Internal</Header1>
+        <div className="grid grid-cols-8 gap-4">
+          {iconNames
+            .sort((a, b) => a.localeCompare(b))
+            .map((iconName) => (
+              <div key={iconName} className="flex items-center gap-2">
+                <div>
+                  <NamedIcon name={iconName} className={"h-6 w-6"} />
+                </div>
+                <span className="text-xs text-dimmed">{iconName}</span>
+              </div>
+            ))}
+        </div>
+      </div>
+      <div>
+        <Header1 spacing>Tabler</Header1>
+        <div className="grid grid-cols-8 gap-4">
+          {Array.from(tablerIcons).map((iconName) => (
+            <div key={iconName} className="flex items-center gap-2">
+              <div>
+                <NamedIcon name={iconName} className={"h-6 w-6 text-indigo-500"} />
+              </div>
+              <span className="text-xs text-dimmed">{iconName}</span>
             </div>
-            <span className="text-xs text-dimmed">{iconName}</span>
-          </div>
-        ))}
+          ))}
+        </div>
+      </div>
     </div>
   );
 }

apps/webapp/app/components/stories/ToastUI.stories.tsx

@@ -0,0 +1,47 @@
+import type { Meta, StoryObj } from "@storybook/react";
+import { Toaster, toast } from "sonner";
+import { ToastUI } from "../primitives/Toast";
+import { Button } from "../primitives/Buttons";
+
+const meta: Meta = {
+  title: "Primitives/Toast",
+};
+
+export default meta;
+
+type Story = StoryObj<typeof Collection>;
+
+export const Toasts: Story = {
+  render: () => <Collection />,
+};
+
+function Collection() {
+  return (
+    <div className="flex flex-col items-start gap-y-4 p-4">
+      <ToastUI variant="success" message="Success UI" t="-" />
+      <ToastUI variant="error" message="Error UI" t="-" />
+      <br />
+      <Button
+        variant="primary/large"
+        onClick={() =>
+          toast.custom((t) => <ToastUI variant="success" message="Success" t={t as string} />, {
+            duration: Infinity, // Prevents auto-dismissal for demo purposes
+          })
+        }
+      >
+        Success
+      </Button>
+      <Button
+        variant="primary/large"
+        onClick={() =>
+          toast.custom((t) => <ToastUI variant="error" message="Error" t={t as string} />, {
+            duration: Infinity,
+          })
+        }
+      >
+        Error
+      </Button>
+      <Toaster />
+    </div>
+  );
+}

apps/webapp/app/consts.ts

@@ -6,3 +6,6 @@ export const MAX_CONCURRENT_RUNS_LIMIT = 20;
 export const PREPROCESS_RETRY_LIMIT = 2;
 export const EXECUTE_JOB_RETRY_LIMIT = 10;
 export const MAX_RUN_YIELDED_EXECUTIONS = 100;
+export const RUN_CHUNK_EXECUTION_BUFFER = 350;
+export const MAX_RUN_CHUNK_EXECUTION_LIMIT = 120000; // 2 minutes
+export const RESPONSE_TIMEOUT_STATUS_CODES = [408, 504];

apps/webapp/app/env.server.ts

@@ -40,6 +40,7 @@ const EnvironmentSchema = z.object({
   EXECUTION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   WORKER_ENABLED: z.string().default("true"),
   EXECUTION_WORKER_ENABLED: z.string().default("true"),
+  GRACEFUL_SHUTDOWN_TIMEOUT: z.coerce.number().int().default(60000),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/hooks/useEnvironments.ts

@@ -1,5 +1,6 @@
 import { RouteMatch } from "@remix-run/react";
 import { MatchedProject, useOptionalProject } from "./useProject";
+import { useUser } from "./useUser";
 
 export type ProjectJobEnvironment = MatchedProject["environments"][number];
 
@@ -11,10 +12,13 @@ export function useEnvironments(matches?: RouteMatch[]) {
 }
 
 export function useDevEnvironment(matches?: RouteMatch[]) {
+  const user = useUser();
   const environments = useEnvironments(matches);
   if (!environments) return;
 
-  return environments.find((environment) => environment.type === "DEVELOPMENT");
+  return environments.find(
+    (environment) => environment.type === "DEVELOPMENT" && environment.userId === user.id
+  );
 }
 
 export function useProdEnvironment(matches?: RouteMatch[]) {

apps/webapp/app/hooks/useFilterJobs.ts

@@ -1,9 +1,21 @@
 import { ProjectJob } from "./useJobs";
 import { useTextFilter } from "./useTextFilter";
+import { useToggleFilter } from "./useToggleFilter";
 
-export function useFilterJobs(jobs: ProjectJob[]) {
-  const { filterText, setFilterText, filteredItems } = useTextFilter<ProjectJob>({
+export function useFilterJobs(jobs: ProjectJob[], onlyActiveJobs = false) {
+  const toggleFilterRes = useToggleFilter<ProjectJob>({
     items: jobs,
+    filter: (job, onlyActiveJobs) => {
+      if (onlyActiveJobs && job.status !== "ACTIVE") {
+        return false;
+      }
+      return true;
+    },
+    defaultValue: onlyActiveJobs,
+  });
+
+  const textFilterRes = useTextFilter<ProjectJob>({
+    items: toggleFilterRes.filteredItems,
     filter: (job, text) => {
       if (job.slug.toLowerCase().includes(text.toLowerCase())) return true;
       if (job.title.toLowerCase().includes(text.toLowerCase())) return true;
@@ -24,5 +36,11 @@ export function useFilterJobs(jobs: ProjectJob[]) {
     },
   });
 
-  return { filterText, setFilterText, filteredItems };
+  return {
+    filteredItems: textFilterRes.filteredItems,
+    filterText: textFilterRes.filterText,
+    setFilterText: textFilterRes.setFilterText,
+    onlyActiveJobs: toggleFilterRes.isToggleActive,
+    setOnlyActiveJobs: toggleFilterRes.setToggleActive,
+  };
 }

apps/webapp/app/hooks/useToggleFilter.ts

@@ -0,0 +1,21 @@
+import { useMemo, useState } from "react";
+
+type ToggleFilterProps<T> = {
+  items: T[];
+  filter: (item: T, isToggleActive: boolean) => boolean;
+  defaultValue?: boolean;
+};
+
+export function useToggleFilter<T>({ items, filter, defaultValue = false }: ToggleFilterProps<T>) {
+  const [isToggleActive, setToggleActive] = useState(defaultValue);
+
+  const filteredItems = useMemo<T[]>(() => {
+    return items.filter((item) => filter(item, isToggleActive));
+  }, [items, isToggleActive]);
+
+  return {
+    isToggleActive,
+    setToggleActive,
+    filteredItems,
+  };
+}

apps/webapp/app/models/endpoint.server.ts

@@ -1,3 +1,4 @@
+import { RESPONSE_TIMEOUT_STATUS_CODES } from "~/consts";
 import { prisma } from "~/db.server";
 import { Prettify } from "~/lib.es5";
 
@@ -18,3 +19,14 @@ export async function findEndpoint(id: string) {
     },
   });
 }
+
+export function detectResponseIsTimeout(response?: Response) {
+  if (!response) {
+    return false;
+  }
+
+  return (
+    RESPONSE_TIMEOUT_STATUS_CODES.includes(response.status) ||
+    response.headers.get("x-vercel-error") === "FUNCTION_INVOCATION_TIMEOUT"
+  );
+}

apps/webapp/app/models/indexEndpoint.server.ts

@@ -1,14 +0,0 @@
-import { z } from "zod";
-
-const IndexEndpointStatsSchema = z.object({
-  jobs: z.number(),
-  sources: z.number(),
-  dynamicTriggers: z.number(),
-  dynamicSchedules: z.number(),
-});
-
-export type IndexEndpointStats = z.infer<typeof IndexEndpointStatsSchema>;
-
-export function parseEndpointIndexStats(stats: unknown): IndexEndpointStats {
-  return IndexEndpointStatsSchema.parse(stats);
-}

apps/webapp/app/models/jobRunExecution.server.ts

@@ -2,27 +2,6 @@ import { JobRun, JobRunExecution } from "@trigger.dev/database";
 import { PrismaClientOrTransaction } from "~/db.server";
 import { executionWorker } from "~/services/worker.server";
 
-export async function enqueueRunExecutionV1(
-  execution: JobRunExecution,
-  queueId: string,
-  concurrency: number,
-  tx: PrismaClientOrTransaction,
-  runAt?: Date
-) {
-  const job = await executionWorker.enqueue(
-    "performRunExecution",
-    {
-      id: execution.id,
-    },
-    {
-      queueName: `job:queue:${queueId}`,
-      tx,
-      runAt,
-      jobKey: `execution:${execution.runId}`,
-    }
-  );
-}
-
 export type EnqueueRunExecutionV2Options = {
   runAt?: Date;
   resumeTaskId?: string;

apps/webapp/app/models/task.server.ts

@@ -1,7 +1,7 @@
-import type { Task, TaskAttempt } from "@trigger.dev/database";
+import type { JobRun, Task, TaskAttempt } from "@trigger.dev/database";
 import { CachedTask, ServerTask } from "@trigger.dev/core";
 
-export type TaskWithAttempts = Task & { attempts: TaskAttempt[] };
+export type TaskWithAttempts = Task & { attempts: TaskAttempt[]; run: JobRun };
 
 export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask {
   return {
@@ -24,6 +24,7 @@ export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask
     idempotencyKey: task.idempotencyKey,
     operation: task.operation,
     callbackUrl: task.callbackUrl,
+    forceYield: task.run.forceYieldImmediately,
   };
 }
 

apps/webapp/app/platform/zodWorker.server.ts

@@ -14,7 +14,7 @@ import { run as graphileRun, parseCronItems } from "graphile-worker";
 import omit from "lodash.omit";
 import { z } from "zod";
 import { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
-import { logger } from "~/services/logger.server";
+import { workerLogger as logger } from "~/services/logger.server";
 
 export interface MessageCatalogSchema {
   [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
@@ -81,6 +81,18 @@ export type ZodWorkerDequeueOptions = {
   tx?: PrismaClientOrTransaction;
 };
 
+const CLEANUP_TASK_NAME = "__cleanupOldJobs";
+const REPORTER_TASK_NAME = "__reporter";
+
+export type ZodWorkerCleanupOptions = {
+  frequencyExpression: string; // cron expression
+  ttl: number;
+  maxCount: number;
+  taskOptions?: CronItemOptions;
+};
+
+type ZodWorkerReporter = (event: string, properties: Record<string, any>) => Promise<void>;
+
 export type ZodWorkerOptions<TMessageCatalog extends MessageCatalogSchema> = {
   name: string;
   runnerOptions: RunnerOptions;
@@ -88,6 +100,9 @@ export type ZodWorkerOptions<TMessageCatalog extends MessageCatalogSchema> = {
   schema: TMessageCatalog;
   tasks: ZodTasks<TMessageCatalog>;
   recurringTasks?: ZodRecurringTasks;
+  cleanup?: ZodWorkerCleanupOptions;
+  reporter?: ZodWorkerReporter;
+  shutdownTimeoutInMs?: number;
 };
 
 export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
@@ -98,6 +113,10 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
   #tasks: ZodTasks<TMessageCatalog>;
   #recurringTasks?: ZodRecurringTasks;
   #runner?: GraphileRunner;
+  #cleanup: ZodWorkerCleanupOptions | undefined;
+  #reporter?: ZodWorkerReporter;
+  #shutdownTimeoutInMs?: number;
+  #shuttingDown = false;
 
   constructor(options: ZodWorkerOptions<TMessageCatalog>) {
     this.#name = options.name;
@@ -106,6 +125,9 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     this.#runnerOptions = options.runnerOptions;
     this.#tasks = options.tasks;
     this.#recurringTasks = options.recurringTasks;
+    this.#cleanup = options.cleanup;
+    this.#reporter = options.reporter;
+    this.#shutdownTimeoutInMs = options.shutdownTimeoutInMs ?? 60000; // default to 60 seconds
   }
 
   get graphileWorkerSchema() {
@@ -125,6 +147,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     this.#runner = await graphileRun({
       ...this.#runnerOptions,
+      noHandleSignals: true,
       taskList: this.#createTaskListFromTasks(),
       parsedCronItems,
     });
@@ -181,9 +204,34 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       this.#logDebug("stop");
     });
 
+    process.on("SIGTERM", this._handleSignal.bind(this));
+    process.on("SIGINT", this._handleSignal.bind(this));
+
     return true;
   }
 
+  private _handleSignal(signal: string) {
+    if (this.#shuttingDown) {
+      return;
+    }
+
+    this.#shuttingDown = true;
+
+    if (this.#shutdownTimeoutInMs) {
+      setTimeout(() => {
+        this.#logDebug("Shutdown timeout reached, exiting process");
+
+        process.exit(0);
+      }, this.#shutdownTimeoutInMs);
+    }
+
+    this.#logDebug(`Received ${signal}, shutting down zodWorker...`);
+
+    this.stop().finally(() => {
+      this.#logDebug("zodWorker stopped");
+    });
+  }
+
   public async stop() {
     await this.#runner?.stop();
   }
@@ -337,12 +385,45 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       taskList[key] = task;
     }
 
+    if (this.#cleanup) {
+      const task: Task = (payload, helpers) => {
+        return this.#handleCleanup(payload, helpers);
+      };
+
+      taskList[CLEANUP_TASK_NAME] = task;
+    }
+
+    if (this.#reporter) {
+      const task: Task = (payload, helpers) => {
+        return this.#handleReporter(payload, helpers);
+      };
+
+      taskList[REPORTER_TASK_NAME] = task;
+    }
+
     return taskList;
   }
 
   #createCronItemsFromRecurringTasks() {
     const cronItems: CronItem[] = [];
 
+    if (this.#cleanup) {
+      cronItems.push({
+        pattern: this.#cleanup.frequencyExpression,
+        identifier: CLEANUP_TASK_NAME,
+        task: CLEANUP_TASK_NAME,
+        options: this.#cleanup.taskOptions,
+      });
+    }
+
+    if (this.#reporter) {
+      cronItems.push({
+        pattern: "50 * * * *", // Every hour at 50 minutes past the hour
+        identifier: REPORTER_TASK_NAME,
+        task: REPORTER_TASK_NAME,
+      });
+    }
+
     if (!this.#recurringTasks) {
       return cronItems;
     }
@@ -434,6 +515,112 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     }
   }
 
+  async #handleCleanup(rawPayload: unknown, helpers: JobHelpers): Promise<void> {
+    if (!this.#cleanup) {
+      return;
+    }
+
+    const job = helpers.job;
+
+    logger.debug("Received cleanup task", {
+      payload: rawPayload,
+      job,
+    });
+
+    const parsedPayload = RawCronPayloadSchema.safeParse(rawPayload);
+
+    if (!parsedPayload.success) {
+      throw new Error(
+        `Failed to parse cleanup task payload: ${JSON.stringify(parsedPayload.error)}`
+      );
+    }
+
+    const payload = parsedPayload.data;
+
+    // Add the this.#cleanup.ttl to the payload._cron.ts
+    const expirationDate = new Date(payload._cron.ts.getTime() - this.#cleanup.ttl);
+
+    logger.debug("Cleaning up old jobs", {
+      expirationDate,
+      payload,
+    });
+
+    const rawResults = await this.#prisma.$queryRawUnsafe(
+      `WITH rows AS (SELECT id FROM ${this.graphileWorkerSchema}.jobs WHERE run_at < $1 AND locked_at IS NULL AND max_attempts = attempts LIMIT $2 FOR UPDATE) DELETE FROM ${this.graphileWorkerSchema}.jobs WHERE id IN (SELECT id FROM rows) RETURNING id`,
+      expirationDate,
+      this.#cleanup.maxCount
+    );
+
+    const results = Array.isArray(rawResults) ? rawResults : [];
+
+    logger.debug("Cleaned up old jobs", {
+      count: results.length,
+      expirationDate,
+      payload,
+    });
+
+    if (this.#reporter) {
+      await this.#reporter("cleanup_stats", {
+        count: results.length,
+        expirationDate,
+        ts: payload._cron.ts,
+      });
+    }
+  }
+
+  async #handleReporter(rawPayload: unknown, helpers: JobHelpers): Promise<void> {
+    if (!this.#reporter) {
+      return;
+    }
+
+    logger.debug("Received reporter task", {
+      payload: rawPayload,
+    });
+
+    const parsedPayload = RawCronPayloadSchema.safeParse(rawPayload);
+
+    if (!parsedPayload.success) {
+      throw new Error(
+        `Failed to parse cleanup task payload: ${JSON.stringify(parsedPayload.error)}`
+      );
+    }
+
+    const payload = parsedPayload.data;
+
+    // Subtract an hour from the payload._cron.ts
+    const startAt = new Date(payload._cron.ts.getTime() - 1000 * 60 * 60);
+
+    const schema = z.array(z.object({ count: z.coerce.number() }));
+
+    // Count the number of jobs that have been added since the startAt date and before the payload._cron.ts date
+    const rawAddedResults = await this.#prisma.$queryRawUnsafe(
+      `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs WHERE created_at > $1 AND created_at < $2`,
+      startAt,
+      payload._cron.ts
+    );
+
+    const addedCountResults = schema.parse(rawAddedResults)[0];
+
+    // Count the total number of jobs in the jobs table
+    const rawTotalResults = await this.#prisma.$queryRawUnsafe(
+      `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs`
+    );
+
+    const totalCountResults = schema.parse(rawTotalResults)[0];
+
+    logger.debug("Calculated metrics about the jobs table", {
+      rawAddedResults,
+      rawTotalResults,
+      payload,
+    });
+
+    await this.#reporter("queue_metrics", {
+      addedCount: addedCountResults.count,
+      totalCount: totalCountResults.count,
+      ts: payload._cron.ts,
+    });
+  }
+
   #logDebug(message: string, args?: any) {
     logger.debug(`[worker][${this.#name}] ${message}`, args);
   }

apps/webapp/app/presenters/EnvironmentsPresenter.server.ts

@@ -1,13 +1,19 @@
 import { PrismaClient, prisma } from "~/db.server";
-import { IndexEndpointStats, parseEndpointIndexStats } from "~/models/indexEndpoint.server";
 import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
 import type {
   Endpoint,
   EndpointIndex,
+  EndpointIndexStatus,
   RuntimeEnvironment,
   RuntimeEnvironmentType,
 } from "@trigger.dev/database";
+import {
+  EndpointIndexError,
+  EndpointIndexErrorSchema,
+  IndexEndpointStats,
+  parseEndpointIndexStats,
+} from "@trigger.dev/core";
 
 export type Client = {
   slug: string;
@@ -34,9 +40,11 @@ export type ClientEndpoint =
       url: string;
       indexWebhookPath: string;
       latestIndex?: {
+        status: EndpointIndexStatus;
         source: string;
         updatedAt: Date;
-        stats: IndexEndpointStats;
+        stats?: IndexEndpointStats;
+        error?: EndpointIndexError;
       };
       environment: {
         id: string;
@@ -81,9 +89,11 @@ export class EnvironmentsPresenter {
             indexingHookIdentifier: true,
             indexings: {
               select: {
+                status: true,
                 source: true,
                 updatedAt: true,
                 stats: true,
+                error: true,
               },
               take: 1,
               orderBy: {
@@ -214,7 +224,7 @@ const environmentSortOrder: RuntimeEnvironmentType[] = [
 
 function endpointClient(
   endpoint: Pick<Endpoint, "id" | "slug" | "url" | "indexingHookIdentifier"> & {
-    indexings: Pick<EndpointIndex, "source" | "updatedAt" | "stats">[];
+    indexings: Pick<EndpointIndex, "status" | "source" | "updatedAt" | "stats" | "error">[];
   },
   environment: Pick<RuntimeEnvironment, "id" | "apiKey" | "type">,
   baseUrl: string
@@ -227,9 +237,13 @@ function endpointClient(
     indexWebhookPath: `${baseUrl}/api/v1/endpoints/${environment.id}/${endpoint.slug}/index/${endpoint.indexingHookIdentifier}`,
     latestIndex: endpoint.indexings[0]
       ? {
+          status: endpoint.indexings[0].status,
           source: endpoint.indexings[0].source,
           updatedAt: endpoint.indexings[0].updatedAt,
           stats: parseEndpointIndexStats(endpoint.indexings[0].stats),
+          error: endpoint.indexings[0].error
+            ? EndpointIndexErrorSchema.parse(endpoint.indexings[0].error)
+            : undefined,
         }
       : undefined,
     environment: environment,

apps/webapp/app/presenters/OrgUsagePresenter.server.ts

@@ -1,4 +1,5 @@
 import { PrismaClient, prisma } from "~/db.server";
+import { logger } from "~/services/logger.server";
 
 export class OrgUsagePresenter {
   #prismaClient: PrismaClient;
@@ -33,6 +34,7 @@ export class OrgUsagePresenter {
         createdAt: {
           gte: new Date(new Date().getFullYear(), new Date().getMonth(), 1),
         },
+        internal: false,
       },
     });
 
@@ -44,6 +46,7 @@ export class OrgUsagePresenter {
           gte: startOfLastMonth,
           lt: startOfMonth,
         },
+        internal: false,
       },
     });
 
@@ -63,7 +66,7 @@ export class OrgUsagePresenter {
         month: string;
         count: number;
       }[]
-    >`SELECT TO_CHAR("createdAt", 'YYYY-MM') as month, COUNT(*) as count FROM "JobRun" WHERE "organizationId" = ${organization.id} AND "createdAt" >= NOW() - INTERVAL '6 months' GROUP BY month ORDER BY month ASC`;
+    >`SELECT TO_CHAR("createdAt", 'YYYY-MM') as month, COUNT(*) as count FROM "JobRun" WHERE "organizationId" = ${organization.id} AND "createdAt" >= NOW() - INTERVAL '6 months' AND "internal" = FALSE GROUP BY month ORDER BY month ASC`;
 
     const chartData = chartDataRaw.map((obj) => ({
       name: obj.month,
@@ -139,11 +142,13 @@ export class OrgUsagePresenter {
       },
     });
 
+    const chartDataDisplay = fillInMissingMonthlyData(chartData, 6);
+
     return {
       id: organization.id,
       runsCount,
       runsCountLastMonth,
-      chartData: fillInMissingMonthlyData(chartData, 6),
+      chartData: chartDataDisplay,
       totalJobs,
       totalJobsLastMonth,
       totalIntegrations,
@@ -166,7 +171,7 @@ function fillInMissingMonthlyData(
 
   const startMonth = new Date(
     new Date(currentMonth).getFullYear(),
-    new Date(currentMonth).getMonth() - totalNumberOfMonths,
+    new Date(currentMonth).getMonth() - (totalNumberOfMonths - 2),
     1
   )
     .toISOString()
@@ -182,17 +187,36 @@ function fillInMissingMonthlyData(
   return completeData;
 }
 
+// Start month will be like 2023-03 and endMonth will be like 2023-10
+// The result should be an array of months between these two months, including the start and end month
+// So for example, if startMonth is 2023-03 and endMonth is 2023-10, the result should be:
+// ["2023-03", "2023-04", "2023-05", "2023-06", "2023-07", "2023-08", "2023-09", "2023-10"]
 function getMonthsBetween(startMonth: string, endMonth: string): string[] {
-  const startDate = new Date(startMonth);
-  const endDate = new Date(endMonth);
+  // Initialize result array
+  const result: string[] = [];
 
-  const months = [];
-  let currentDate = startDate;
+  // Parse the year and month from startMonth and endMonth
+  let [startYear, startMonthNum] = startMonth.split("-").map(Number);
+  let [endYear, endMonthNum] = endMonth.split("-").map(Number);
 
-  while (currentDate <= endDate) {
-    months.push(currentDate.toISOString().slice(0, 7));
-    currentDate = new Date(currentDate.setMonth(currentDate.getMonth() + 1));
+  // Loop through each month between startMonth and endMonth
+  for (let year = startYear; year <= endYear; year++) {
+    let monthStart = year === startYear ? startMonthNum : 1;
+    let monthEnd = year === endYear ? endMonthNum : 12;
+
+    for (let month = monthStart; month <= monthEnd; month++) {
+      // Format the month into a string and add it to the result array
+      result.push(`${year}-${String(month).padStart(2, "0")}`);
+    }
   }
 
-  return months;
+  return result;
+}
+
+function getLastSecondOfMonth(endMonth: string) {
+  const [year, month] = endMonth.split("-").map(Number);
+  const nextMonthFirstDay = new Date(year, month, 1);
+  nextMonthFirstDay.setDate(0);
+  nextMonthFirstDay.setHours(23, 59, 59);
+  return nextMonthFirstDay;
 }

apps/webapp/app/presenters/RunListPresenter.server.ts

@@ -100,21 +100,24 @@ export class RunListPresenter {
     let previous: string | undefined;
     switch (direction) {
       case "forward":
+        previous = cursor ? runs.at(0)?.id : undefined;
         if (hasMore) {
           next = runs[PAGE_SIZE - 1]?.id;
         }
-        previous = cursor ? runs.at(1)?.id : undefined;
         break;
       case "backward":
         if (hasMore) {
-          next = runs[PAGE_SIZE - 1]?.id;
+          previous = runs[1]?.id;
         }
-        previous = runs.at(1)?.id;
+        next = runs[PAGE_SIZE - 1]?.id;
         break;
     }
 
+    const runsToReturn =
+      direction === "backward" && hasMore ? runs.slice(1, PAGE_SIZE + 1) : runs.slice(0, PAGE_SIZE);
+
     return {
-      runs: runs.slice(0, PAGE_SIZE).map((run) => ({
+      runs: runsToReturn.map((run) => ({
         id: run.id,
         number: run.number,
         startedAt: run.startedAt,

apps/webapp/app/root.tsx

@@ -25,7 +25,7 @@ export const links: LinksFunction = () => {
 export const meta: TypedMetaFunction<typeof loader> = ({ data }) => ({
   title: `Trigger.dev${appEnvTitleTag(data?.appEnv)}`,
   charset: "utf-8",
-  viewport: "width=device-width,initial-scale=1",
+  viewport: "width=1024, initial-scale=1",
 });
 
 export const loader = async ({ request }: LoaderArgs) => {

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam._index/route.tsx

@@ -19,6 +19,7 @@ import {
   PageTitleRow,
 } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Switch } from "~/components/primitives/Switch";
 import { TextLink } from "~/components/primitives/TextLink";
 import { useFilterJobs } from "~/hooks/useFilterJobs";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -62,8 +63,11 @@ export default function Page() {
   const organization = useOrganization();
   const project = useProject();
   const { jobs } = useTypedLoaderData<typeof loader>();
-  const { filterText, setFilterText, filteredItems } = useFilterJobs(jobs);
-  const hasJobs = jobs.length > 0;
+  const { filterText, setFilterText, filteredItems, onlyActiveJobs, setOnlyActiveJobs } =
+    useFilterJobs(jobs);
+  const totalJobs = jobs.length;
+  const hasJobs = totalJobs > 0;
+  const activeJobCount = jobs.filter((j) => j.status === "ACTIVE").length;
 
   return (
     <PageContainer className={hasJobs ? "" : "grid-rows-1"}>
@@ -74,7 +78,8 @@ export default function Page() {
           </PageTitleRow>
           <PageInfoRow>
             <PageInfoGroup>
-              <PageInfoProperty icon={"job"} label={"Active Jobs"} value={jobs.length} />
+              <PageInfoProperty icon={"job"} label={"All Jobs"} value={totalJobs} />
+              <PageInfoProperty icon={"job"} label={"Active Jobs"} value={activeJobCount} />
             </PageInfoGroup>
           </PageInfoRow>
         </PageHeader>
@@ -96,7 +101,7 @@ export default function Page() {
                       </Callout>
                     )}
                     <div className="mb-2 flex flex-col">
-                      <div className="flex w-full">
+                      <div className="flex w-full gap-x-2">
                         <Input
                           placeholder="Search Jobs"
                           variant="tertiary"
@@ -106,6 +111,13 @@ export default function Page() {
                           onChange={(e) => setFilterText(e.target.value)}
                           autoFocus
                         />
+                        <Switch
+                          variant="small"
+                          label="Active Jobs"
+                          checked={onlyActiveJobs}
+                          onCheckedChange={setOnlyActiveJobs}
+                          className={"shrink-0"}
+                        />
                         <HelpTrigger title="Example Jobs and inspiration" />
                       </div>
                     </div>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx

@@ -6,7 +6,7 @@ import { useEventSource } from "remix-utils";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { Button } from "~/components/primitives/Buttons";
-import { Callout } from "~/components/primitives/Callout";
+import { Callout, CalloutVariant } from "~/components/primitives/Callout";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DateTime } from "~/components/primitives/DateTime";
 import { FormError } from "~/components/primitives/FormError";
@@ -18,8 +18,14 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { Sheet, SheetBody, SheetContent, SheetHeader } from "~/components/primitives/Sheet";
 import { ClientEndpoint } from "~/presenters/EnvironmentsPresenter.server";
 import { endpointStreamingPath } from "~/utils/pathBuilder";
-import { RuntimeEnvironmentType } from "../../../../../packages/database/src";
+import { EndpointIndexStatus, RuntimeEnvironmentType } from "../../../../../packages/database/src";
 import { bodySchema } from "../resources.environments.$environmentParam.endpoint";
+import {
+  EndpointIndexStatusIcon,
+  EndpointIndexStatusLabel,
+  endpointIndexStatusTitle,
+} from "~/components/environments/EndpointIndexStatus";
+import { CodeBlock } from "~/components/code/CodeBlock";
 
 type ConfigureEndpointSheetProps = {
   slug: string;
@@ -119,15 +125,29 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd
                   method="post"
                   action={`/resources/environments/${endpoint.environment.id}/endpoint/${endpoint.id}`}
                 >
-                  <Callout variant="success" className="justiy-between items-center">
-                    <Paragraph variant="small" className="grow text-green-200">
-                      Endpoint configured. Last refreshed:{" "}
-                      {endpoint.latestIndex ? (
-                        <DateTime date={endpoint.latestIndex.updatedAt} />
-                      ) : (
-                        "–"
-                      )}
-                    </Paragraph>
+                  <Callout
+                    variant="info"
+                    icon={
+                      <EndpointIndexStatusIcon status={endpoint.latestIndex?.status ?? "PENDING"} />
+                    }
+                    className="justiy-between items-center"
+                  >
+                    <div className="flex grow items-center gap-2">
+                      <EndpointIndexStatusLabel
+                        status={endpoint.latestIndex?.status ?? "PENDING"}
+                      />
+                      <Paragraph variant="small" className="grow">
+                        Last refreshed:{" "}
+                        {endpoint.latestIndex ? (
+                          <>
+                            <DateTime date={endpoint.latestIndex.updatedAt} />
+                          </>
+                        ) : (
+                          "–"
+                        )}
+                      </Paragraph>
+                    </div>
+
                     <Button
                       variant="primary/small"
                       type="submit"
@@ -138,6 +158,11 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd
                       {refreshingEndpoint ? "Refreshing" : "Refresh now"}
                     </Button>
                   </Callout>
+                  {endpoint.latestIndex?.error && (
+                    <FormError className="p-2">
+                      <pre>{endpoint.latestIndex.error.message}</pre>
+                    </FormError>
+                  )}
                 </refreshEndpointFetcher.Form>
               </div>
               <div className="max-w-full overflow-hidden">
@@ -155,3 +180,16 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd
     </Sheet>
   );
 }
+
+function calloutVariantFromStatus(status: EndpointIndexStatus): CalloutVariant {
+  switch (status) {
+    case "PENDING":
+      return "pending";
+    case "STARTED":
+      return "pending";
+    case "SUCCESS":
+      return "success";
+    case "FAILURE":
+      return "error";
+  }
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx

@@ -3,11 +3,16 @@ import { LoaderArgs } from "@remix-run/server-runtime";
 import { useEffect, useMemo, useState } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { useEventSource } from "remix-utils";
+import {
+  EndpointIndexStatusIcon,
+  EndpointIndexStatusLabel,
+} from "~/components/environments/EndpointIndexStatus";
 import { EnvironmentLabel, environmentTitle } from "~/components/environments/EnvironmentLabel";
 import { HowToUseApiKeysAndEndpoints } from "~/components/helpContent/HelpContentText";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { BreadcrumbLink } from "~/components/navigation/NavBar";
-import { Button, ButtonContent } from "~/components/primitives/Buttons";
+import { Badge } from "~/components/primitives/Badge";
+import { ButtonContent } from "~/components/primitives/Buttons";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DateTime } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
@@ -38,7 +43,6 @@ import { ProjectParamSchema, projectEnvironmentsStreamingPath } from "~/utils/pa
 import { requestUrl } from "~/utils/requestUrl.server";
 import { RuntimeEnvironmentType } from "../../../../../packages/database/src";
 import { ConfigureEndpointSheet } from "./ConfigureEndpointSheet";
-import { Badge } from "~/components/primitives/Badge";
 import { FirstEndpointSheet } from "./FirstEndpointSheet";
 
 export const loader = async ({ request, params }: LoaderArgs) => {
@@ -180,6 +184,7 @@ export default function Page() {
                               <TableHeaderCell>Environment</TableHeaderCell>
                               <TableHeaderCell>Url</TableHeaderCell>
                               <TableHeaderCell>Last refreshed</TableHeaderCell>
+                              <TableHeaderCell>Last refresh Status</TableHeaderCell>
                               <TableHeaderCell>Jobs</TableHeaderCell>
                               <TableHeaderCell hiddenLabel>Go to page</TableHeaderCell>
                             </TableRow>
@@ -268,7 +273,7 @@ function EndpointRow({
               <EnvironmentLabel environment={{ type }} />
             </div>
           </TableCell>
-          <TableCell onClick={onClick} colSpan={4} alignment="right">
+          <TableCell onClick={onClick} colSpan={5} alignment="right">
             <div className="flex items-center justify-end gap-4">
               <span className="text-amber-500">
                 The {environmentTitle({ type })} environment is not configured
@@ -290,7 +295,17 @@ function EndpointRow({
           <TableCell onClick={onClick}>
             {endpoint.latestIndex ? <DateTime date={endpoint.latestIndex.updatedAt} /> : "–"}
           </TableCell>
-          <TableCell onClick={onClick}>{endpoint.latestIndex?.stats.jobs ?? "–"}</TableCell>
+          <TableCell onClick={onClick}>
+            {endpoint.latestIndex ? (
+              <div className="flex items-center gap-1">
+                <EndpointIndexStatusIcon status={endpoint.latestIndex.status} />
+                <EndpointIndexStatusLabel status={endpoint.latestIndex.status} />
+              </div>
+            ) : (
+              "–"
+            )}
+          </TableCell>
+          <TableCell onClick={onClick}>{endpoint.latestIndex?.stats?.jobs ?? "–"}</TableCell>
           <TableCellChevron onClick={onClick} />
         </TableRow>
       );

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.integrations/route.tsx

@@ -152,7 +152,7 @@ function PossibleIntegrationsList({
           onCheckedChange={setOnlyShowIntegrations}
           variant="small"
           label={
-            <span className="inline-flex items-center gap-1">
+            <span className="mt-0.5 inline-flex items-center gap-1">
               <IntegrationIcon /> Trigger.dev Integrations
             </span>
           }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.integrations_.$clientParam._index/route.tsx

@@ -8,6 +8,7 @@ import { Callout } from "~/components/primitives/Callout";
 import { Header2 } from "~/components/primitives/Headers";
 import { Help, HelpContent, HelpTrigger } from "~/components/primitives/Help";
 import { Input } from "~/components/primitives/Input";
+import { Paragraph } from "~/components/primitives/Paragraph";
 import { useFilterJobs } from "~/hooks/useFilterJobs";
 import { useIntegrationClient } from "~/hooks/useIntegrationClient";
 import { JobListPresenter } from "~/presenters/JobListPresenter.server";
@@ -52,10 +53,8 @@ export default function Page() {
       {(open) => (
         <div className={cn("grid h-full gap-4", open ? "grid-cols-2" : "grid-cols-1")}>
           <div className="grow">
-            <div className="mb-2 flex items-center justify-between gap-x-2">
-              {jobs.length === 0 ? (
-                <Header2>Jobs using this integration will appear here</Header2>
-              ) : (
+            <div className="mb-2 flex items-center justify-end gap-x-2">
+              {jobs.length !== 0 && (
                 <Input
                   placeholder="Search Jobs"
                   variant="tertiary"
@@ -68,9 +67,9 @@ export default function Page() {
               <HelpTrigger title="How do I use this integration?" />
             </div>
             {jobs.length === 0 ? (
-              <>
-                <JobSkeleton />
-              </>
+              <div className="mt-8 rounded border border-border px-2 py-6 text-center">
+                <Paragraph variant="small">Jobs using this Integration will appear here.</Paragraph>
+              </div>
             ) : (
               <JobsTable
                 jobs={filteredItems}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.test/route.tsx

@@ -35,6 +35,7 @@ import { TestJobService } from "~/services/jobs/testJob.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { Handle } from "~/utils/handle";
+import { isValidIcon } from "~/utils/icon";
 import { JobParamsSchema, jobRunDashboardPath, trimTrailingSlash } from "~/utils/pathBuilder";
 
 export const loader = async ({ request, params }: LoaderArgs) => {
@@ -228,7 +229,7 @@ export default function Page() {
                       }}
                     >
                       <DetailCell
-                        leadingIcon={example.icon ?? CodeBracketIcon}
+                        leadingIcon={isValidIcon(example.icon) ? example.icon : CodeBracketIcon}
                         leadingIconClassName="text-blue-500"
                         label={example.name}
                         trailingIcon={example.id === selectedCodeSampleId ? "check" : "plus"}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.setup.sveltekit/route.tsx

@@ -1,23 +1,113 @@
-import { SvelteKitLogo } from "~/assets/logos/SveltekitLogo";
-import { FrameworkComingSoon } from "~/components/frameworks/FrameworkComingSoon";
+import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
+import invariant from "tiny-invariant";
+import { Feedback } from "~/components/Feedback";
+import { PageGradient } from "~/components/PageGradient";
+import { RunDevCommand, TriggerDevStep } from "~/components/SetupCommands";
+import { StepContentContainer } from "~/components/StepContentContainer";
+import { InlineCode } from "~/components/code/InlineCode";
 import { BreadcrumbLink } from "~/components/navigation/NavBar";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { ClipboardField } from "~/components/primitives/ClipboardField";
+import { Header1 } from "~/components/primitives/Headers";
+import { NamedIcon } from "~/components/primitives/NamedIcon";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { StepNumber } from "~/components/primitives/StepNumber";
+import { useAppOrigin } from "~/hooks/useAppOrigin";
+import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
+import { useDevEnvironment } from "~/hooks/useEnvironments";
+import { useOrganization } from "~/hooks/useOrganizations";
+import { useProject } from "~/hooks/useProject";
 import { Handle } from "~/utils/handle";
-import { trimTrailingSlash } from "~/utils/pathBuilder";
-
+import { projectSetupPath, trimTrailingSlash } from "~/utils/pathBuilder";
+import { Callout } from "~/components/primitives/Callout";
+import { Badge } from "~/components/primitives/Badge";
 export const handle: Handle = {
   breadcrumb: (match) => (
     <BreadcrumbLink to={trimTrailingSlash(match.pathname)} title="SvelteKit" />
   ),
 };
 
-export default function Page() {
+export default function SetUpSveltekit() {
+  const organization = useOrganization();
+  const project = useProject();
+  useProjectSetupComplete();
+  const devEnvironment = useDevEnvironment();
+  invariant(devEnvironment, "Dev environment must be defined");
   return (
-    <FrameworkComingSoon
-      frameworkName="SvelteKit"
-      githubIssueUrl="https://github.com/triggerdotdev/trigger.dev/issues/453"
-      githubIssueNumber={453}
-    >
-      <SvelteKitLogo className="w-56" />
-    </FrameworkComingSoon>
+    <PageGradient>
+      <div className="mx-auto max-w-3xl">
+        <div className="flex items-center justify-between">
+          <Header1 spacing className="text-bright">
+            Get setup in 5 minutes
+          </Header1>
+          <div className="flex items-center gap-2">
+            <LinkButton
+              to={projectSetupPath(organization, project)}
+              variant="tertiary/small"
+              LeadingIcon={Squares2X2Icon}
+            >
+              Choose a different framework
+            </LinkButton>
+            <Feedback
+              button={
+                <Button variant="tertiary/small" LeadingIcon={ChatBubbleLeftRightIcon}>
+                  I'm stuck!
+                </Button>
+              }
+              defaultValue="help"
+            />
+          </div>
+        </div>
+        <div>
+          <Callout
+            variant={"info"}
+            to="https://github.com/triggerdotdev/trigger.dev/discussions/430"
+            className="mb-8"
+          >
+            Trigger.dev has full support for serverless. We will be adding support for long-running
+            servers soon.
+          </Callout>
+          <div>
+            <StepNumber
+              stepNumber="1"
+              title="Follow the steps from the Sveltekit manual installation guide"
+            />
+            <StepContentContainer className="flex flex-col gap-2">
+              <Paragraph className="mt-2">Copy your server API Key to your clipboard:</Paragraph>
+              <div className="mb-2 flex w-full items-center justify-between">
+                <ClipboardField
+                  secure
+                  className="w-fit"
+                  value={devEnvironment.apiKey}
+                  variant={"secondary/medium"}
+                  icon={<Badge variant="outline">Server</Badge>}
+                />
+              </div>
+              <Paragraph>Now follow this guide:</Paragraph>
+              <LinkButton
+                to="https://trigger.dev/docs/documentation/guides/manual/sveltekit"
+                variant="primary/medium"
+                TrailingIcon="external-link"
+              >
+                Manual installation guide
+              </LinkButton>
+              <div className="flex items-start justify-start gap-2"></div>
+            </StepContentContainer>
+            <StepNumber stepNumber="2" title="Run your sveltekit app" />
+            <StepContentContainer>
+              <RunDevCommand extra=" -- --open --host" />
+            </StepContentContainer>
+            <StepNumber stepNumber="3" title="Run the CLI 'dev' command" />
+            <StepContentContainer>
+              <TriggerDevStep extra=" --port 5173" />
+            </StepContentContainer>
+            <StepNumber stepNumber="6" title="Wait for Jobs" displaySpinner />
+            <StepContentContainer>
+              <Paragraph>This page will automatically refresh.</Paragraph>
+            </StepContentContainer>
+          </div>
+        </div>
+      </div>
+    </PageGradient>
   );
 }

apps/webapp/app/routes/_app/route.tsx

@@ -60,7 +60,6 @@ export default function App() {
   return (
     <>
       {impersonationId && <ImpersonationBanner impersonationId={impersonationId} />}
-      <NoMobileOverlay />
       <AppContainer showBackgroundGradient={showBackgroundGradient}>
         <NavBar />
         <Outlet />
@@ -72,7 +71,6 @@ export default function App() {
 export function ErrorBoundary() {
   return (
     <>
-      <NoMobileOverlay />
       <AppContainer showBackgroundGradient={true}>
         <MainCenteredContainer>
           <RouteErrorDisplay />

apps/webapp/app/routes/admin._index.tsx

@@ -30,89 +30,80 @@ export async function action({ request }: ActionArgs) {
   });
 }
 
-const headerClassName =
-  "py-3 px-2 pr-3 text-xs font-semibold leading-tight text-slate-900 text-left";
-const cellClassName = "whitespace-nowrap px-2 py-2 text-xs text-slate-500";
+const headerClassName = "py-3 px-2 pr-3 text-xs font-semibold leading-tight text-bright text-left";
+const cellClassName = "whitespace-nowrap px-2 py-2 text-xs text-bright";
 
 export default function AdminDashboardRoute() {
   const { users } = useTypedLoaderData<typeof loader>();
 
   return (
-    <main className="flex flex-1 overflow-hidden">
-      {/* Primary column */}
-      <section
-        aria-labelledby="primary-heading"
-        className="flex h-full min-w-0 flex-1 flex-col overflow-y-auto p-4 lg:order-last"
-      >
-        <h1 className="mb-2 text-2xl">Accounts ({users.length})</h1>
+    <main
+      aria-labelledby="primary-heading"
+      className="flex h-full min-w-0 flex-1 flex-col overflow-y-auto p-4 lg:order-last"
+    >
+      <h1 className="mb-2 text-2xl">Accounts ({users.length})</h1>
 
-        <table className="w-full divide-y divide-slate-300">
-          <thead className="sticky top-0 bg-white text-left outline outline-2 outline-slate-200">
-            <tr>
-              <th scope="col" className={headerClassName}>
-                Email
-              </th>
-              <th scope="col" className={headerClassName}>
-                GitHub username
-              </th>
-              <th scope="col" className={headerClassName}>
-                id
-              </th>
-              <th scope="col" className={headerClassName}>
-                Created At
-              </th>
-              <th scope="col" className={headerClassName}>
-                Admin?
-              </th>
-              <th scope="col" className={headerClassName}>
-                Actions
-              </th>
-            </tr>
-          </thead>
-          <tbody className="divide-y divide-slate-200 bg-white">
-            {users.map((user) => {
-              return (
-                <tr key={user.id} className="w-full bg-white px-4 py-2 text-left hover:bg-slate-50">
-                  <td className={cellClassName}>{user.email}</td>
-                  <td className={cellClassName}>
-                    <a
-                      href={`https://github.com/${user.displayName}`}
-                      target="_blank"
-                      className="text-indigo-500 underline"
-                      rel="noreferrer"
+      <table className="w-full divide-y divide-border">
+        <thead className="sticky -top-4 bg-midnight-800 text-left">
+          <tr>
+            <th scope="col" className={headerClassName}>
+              Email
+            </th>
+            <th scope="col" className={headerClassName}>
+              GitHub username
+            </th>
+            <th scope="col" className={headerClassName}>
+              id
+            </th>
+            <th scope="col" className={headerClassName}>
+              Created At
+            </th>
+            <th scope="col" className={headerClassName}>
+              Admin?
+            </th>
+            <th scope="col" className={headerClassName}>
+              Actions
+            </th>
+          </tr>
+        </thead>
+        <tbody className="divide-y divide-border">
+          {users.map((user) => {
+            return (
+              <tr key={user.id} className="w-full px-4 py-2 text-left hover:bg-slate-900">
+                <td className={cellClassName}>{user.email}</td>
+                <td className={cellClassName}>
+                  <a
+                    href={`https://github.com/${user.displayName}`}
+                    target="_blank"
+                    className="text-indigo-500 underline"
+                    rel="noreferrer"
+                  >
+                    {user.displayName}
+                  </a>
+                </td>
+                <td className={cellClassName}>{user.id}</td>
+                <td className={cellClassName}>{user.createdAt.toISOString()}</td>
+                <td className={cellClassName}>{user.admin ? "✅" : ""}</td>
+                <td className={cellClassName}>
+                  <Form method="post" reloadDocument>
+                    <input type="hidden" name="id" value={user.id} />
+
+                    <Button
+                      type="submit"
+                      name="action"
+                      value="impersonate"
+                      className="mr-2"
+                      variant="primary/small"
                     >
-                      {user.displayName}
-                    </a>
-                  </td>
-                  <td className={cellClassName}>{user.id}</td>
-                  <td className={cellClassName}>{user.createdAt.toISOString()}</td>
-                  <td className={cellClassName}>{user.admin ? "✅" : ""}</td>
-                  <td className={cellClassName}>
-                    <Form method="post" reloadDocument>
-                      <input type="hidden" name="id" value={user.id} />
-
-                      <Button
-                        type="submit"
-                        name="action"
-                        value="impersonate"
-                        className="mr-2"
-                        variant="primary/small"
-                      >
-                        Impersonate
-                      </Button>
-                    </Form>
-                  </td>
-                </tr>
-              );
-            })}
-          </tbody>
-        </table>
-      </section>
-
-      {/* Secondary column (hidden on smaller screens) */}
-      <aside className="hidden lg:order-first lg:block lg:flex-shrink-0">
-        <div className="relative flex h-full w-96 flex-col overflow-y-auto border-r border-gray-200 bg-white"></div>
-      </aside>
+                      Impersonate
+                    </Button>
+                  </Form>
+                </td>
+              </tr>
+            );
+          })}
+        </tbody>
+      </table>
     </main>
   );
 }

apps/webapp/app/routes/admin.api.v1.simulate.ts

@@ -0,0 +1,34 @@
+import { ActionArgs, json, redirect } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { workerQueue } from "~/services/worker.server";
+
+export async function action({ request }: ActionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const adminOrgMembers = await prisma.orgMember.findMany({
+    where: {
+      organizationId: authenticationResult.environment.organizationId,
+      user: {
+        admin: true,
+      },
+    },
+  });
+
+  if (!adminOrgMembers.length) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  const body: any = await request.json();
+
+  await workerQueue.enqueue("simulate", {
+    seconds: body.seconds,
+  });
+
+  return json({ success: true });
+}

apps/webapp/app/routes/admin.tsx

@@ -1,15 +1,8 @@
-import { Dialog, Transition } from "@headlessui/react";
-import { HomeIcon, XMarkIcon } from "@heroicons/react/24/outline";
-import { UserCircleIcon } from "@heroicons/react/24/solid";
+import { HomeIcon } from "@heroicons/react/24/outline";
 import { Outlet } from "@remix-run/react";
 import type { LoaderArgs } from "@remix-run/server-runtime";
-import { Fragment, useState } from "react";
 import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson";
-import type { User } from "~/models/user.server";
 import { getUser, requireUserId } from "~/services/session.server";
-import { cn } from "~/utils/cn";
-
-const navigation = [{ name: "Home", href: "/admin", icon: HomeIcon }];
 
 export async function loader({ request }: LoaderArgs) {
   await requireUserId(request);
@@ -27,162 +20,10 @@ export async function loader({ request }: LoaderArgs) {
 
 export default function Page() {
   const data = useTypedLoaderData<typeof loader>();
-  const [mobileMenuOpen, setMobileMenuOpen] = useState(false);
 
   return (
-    <>
-      <div className="flex h-full">
-        <Transition.Root show={mobileMenuOpen} as={Fragment}>
-          <Dialog as="div" className="relative z-40 lg:hidden" onClose={setMobileMenuOpen}>
-            <Transition.Child
-              as={Fragment}
-              enter="transition-opacity ease-linear duration-300"
-              enterFrom="opacity-0"
-              enterTo="opacity-100"
-              leave="transition-opacity ease-linear duration-300"
-              leaveFrom="opacity-100"
-              leaveTo="opacity-0"
-            >
-              <div className="fixed inset-0 bg-gray-600 bg-opacity-75" />
-            </Transition.Child>
-
-            <div className="fixed inset-0 z-40 flex">
-              <Transition.Child
-                as={Fragment}
-                enter="transition ease-in-out duration-300 transform"
-                enterFrom="-translate-x-full"
-                enterTo="translate-x-0"
-                leave="transition ease-in-out duration-300 transform"
-                leaveFrom="translate-x-0"
-                leaveTo="-translate-x-full"
-              >
-                <Dialog.Panel className="relative flex w-full max-w-xs flex-1 flex-col bg-white focus:outline-none">
-                  <Transition.Child
-                    as={Fragment}
-                    enter="ease-in-out duration-300"
-                    enterFrom="opacity-0"
-                    enterTo="opacity-100"
-                    leave="ease-in-out duration-300"
-                    leaveFrom="opacity-100"
-                    leaveTo="opacity-0"
-                  >
-                    <div className="absolute top-0 right-0 -mr-12 pt-4">
-                      <button
-                        type="button"
-                        className="ml-1 flex h-10 w-10 items-center justify-center rounded-full focus:outline-none focus:ring-2 focus:ring-inset focus:ring-white"
-                        onClick={() => setMobileMenuOpen(false)}
-                      >
-                        <span className="sr-only">Close sidebar</span>
-                        <XMarkIcon className="h-6 w-6 text-white" aria-hidden="true" />
-                      </button>
-                    </div>
-                  </Transition.Child>
-                  <div className="pt-5 pb-4">
-                    <div className="flex flex-shrink-0 items-center px-4">
-                      <img
-                        className="h-8 w-auto"
-                        src="https://tailwindui.com/img/logos/workflow-mark.svg?color=indigo&shade=600"
-                        alt="Workflow"
-                      />
-                    </div>
-                    <nav aria-label="Sidebar" className="mt-5">
-                      <div className="space-y-1 px-2">
-                        {navigation.map((item) => (
-                          <a
-                            key={item.name}
-                            href={item.href}
-                            className="group flex items-center rounded-md p-2 text-base font-medium text-gray-600 hover:bg-gray-50 hover:text-gray-900"
-                          >
-                            <item.icon
-                              className="mr-4 h-6 w-6 text-gray-400 group-hover:text-gray-500"
-                              aria-hidden="true"
-                            />
-                            {item.name}
-                          </a>
-                        ))}
-                      </div>
-                    </nav>
-                  </div>
-                  <div className="flex flex-shrink-0 border-t border-gray-200 p-4">
-                    <button className="group block flex-shrink-0">
-                      <div className="flex items-center">
-                        <div>
-                          <UserProfilePhoto user={data.user} className="h-10 w-10" />
-                        </div>
-                        <div className="ml-3">
-                          <p className="text-base font-medium text-gray-700 group-hover:text-gray-900">
-                            {data.user.displayName}
-                          </p>
-                          <p className="text-sm font-medium text-gray-500 group-hover:text-gray-700">
-                            Account Settings
-                          </p>
-                        </div>
-                      </div>
-                    </button>
-                  </div>
-                </Dialog.Panel>
-              </Transition.Child>
-              <div className="w-14 flex-shrink-0" aria-hidden="true">
-                {/* Force sidebar to shrink to fit close icon */}
-              </div>
-            </div>
-          </Dialog>
-        </Transition.Root>
-
-        {/* Static sidebar for desktop */}
-        <div className="hidden lg:flex lg:flex-shrink-0">
-          <div className="flex w-20 flex-col">
-            <div className="flex min-h-0 flex-1 flex-col overflow-y-auto bg-indigo-600">
-              <div className="flex-1">
-                <div className="flex items-center justify-center bg-indigo-700 py-4">
-                  <img
-                    className="h-8 w-auto"
-                    src="https://tailwindui.com/img/logos/workflow-mark.svg?color=white"
-                    alt="Workflow"
-                  />
-                </div>
-                <nav aria-label="Sidebar" className="flex flex-col items-center space-y-3 py-6">
-                  {navigation.map((item) => (
-                    <a
-                      key={item.name}
-                      href={item.href}
-                      className="flex items-center rounded-lg p-4 text-indigo-200 hover:bg-indigo-700"
-                    >
-                      <item.icon className="h-6 w-6" aria-hidden="true" />
-                      <span className="sr-only">{item.name}</span>
-                    </a>
-                  ))}
-                </nav>
-              </div>
-              <div className="flex flex-shrink-0 pb-5">
-                <button className="flex w-full flex-shrink-0 flex-grow justify-center">
-                  <UserProfilePhoto user={data.user} className="block h-10 w-10" />
-                  <div className="sr-only">
-                    <p>{data.user.displayName}</p>
-                    <p>Account settings</p>
-                  </div>
-                </button>
-              </div>
-            </div>
-          </div>
-        </div>
-
-        <div className="flex min-w-0 flex-1 flex-col overflow-hidden">
-          <Outlet />
-        </div>
-      </div>
-    </>
-  );
-}
-
-function UserProfilePhoto({ user, className }: { user: User; className?: string }) {
-  return user.avatarUrl ? (
-    <img
-      className={cn("rounded-full", className)}
-      src={user.avatarUrl}
-      alt={user.name ?? user.displayName ?? "User"}
-    />
-  ) : (
-    <UserCircleIcon className={cn("text-gray-400", className)} />
+    <div className="h-full w-full">
+      <Outlet />
+    </div>
   );
 }

apps/webapp/app/routes/api.v1.endpointindex.$indexId.ts

@@ -0,0 +1,68 @@
+import { ActionArgs, json } from "@remix-run/server-runtime";
+import {
+  EndpointIndexErrorSchema,
+  GetEndpointIndexResponse,
+  GetEndpointIndexResponseSchema,
+} from "@trigger.dev/core";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { IndexEndpointService } from "~/services/endpoints/indexEndpoint.server";
+import { logger } from "~/services/logger.server";
+
+const ParamsSchema = z.object({
+  indexId: z.string(),
+});
+
+export async function loader({ request, params }: ActionArgs) {
+  if (request.method.toUpperCase() !== "GET") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+  if (!authenticationResult) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  const { indexId } = parsedParams.data;
+
+  const endpointIndex = await prisma.endpointIndex.findUnique({
+    where: {
+      id: indexId,
+      endpoint: {
+        environmentId: authenticatedEnv.id,
+      },
+    },
+  });
+
+  if (!endpointIndex) {
+    logger.info("EndpointIndex not found", { url: request.url });
+    return json({ error: "EndpointIndex not found" }, { status: 404 });
+  }
+
+  const parsed = GetEndpointIndexResponseSchema.safeParse(endpointIndex);
+
+  if (!parsed.success) {
+    logger.info("EndpointIndex failed parsing", { errors: parsed.error.issues, endpointIndex });
+    const parseFailResult: GetEndpointIndexResponse = {
+      status: "FAILURE",
+      error: {
+        message: "Invalid endpoint index",
+      },
+      updatedAt: new Date(),
+    };
+    return json(parseFailResult, { status: 500 });
+  }
+
+  return json(parsed.data);
+}

apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier.ts

@@ -1,7 +1,7 @@
 import { ActionArgs, LoaderArgs, json } from "@remix-run/server-runtime";
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { z } from "zod";
-import { PrismaClient, prisma } from "~/db.server";
+import { $transaction, PrismaClient, prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { workerQueue } from "~/services/worker.server";
 import { safeJsonParse } from "~/utils/json";
@@ -93,43 +93,53 @@ export class TriggerEndpointIndexHookService {
       body,
     });
 
-    const endpoint = await this.#prismaClient.endpoint.findUnique({
-      where: {
-        environmentId_slug: {
-          environmentId,
-          slug: endpointSlug,
+    await $transaction(this.#prismaClient, async (tx) => {
+      const endpoint = await tx.endpoint.findUnique({
+        where: {
+          environmentId_slug: {
+            environmentId,
+            slug: endpointSlug,
+          },
         },
-      },
-      include: {
-        environment: true,
-      },
-    });
+        include: {
+          environment: true,
+        },
+      });
 
-    if (!endpoint) {
-      throw new Error("Endpoint not found");
-    }
-
-    if (endpoint.indexingHookIdentifier !== indexHookIdentifier) {
-      throw new Error("Index hook identifier is invalid");
-    }
-
-    const reason = parseReasonFromBody(body);
-
-    // Index the endpoint in 5 seconds from now
-    await workerQueue.enqueue(
-      "indexEndpoint",
-      {
-        id: endpoint.id,
-        source: "HOOK",
-        reason,
-        sourceData: body,
-      },
-      {
-        runAt: new Date(Date.now() + 5000),
-        maxAttempts:
-          endpoint.environment.type === RuntimeEnvironmentType.DEVELOPMENT ? 1 : undefined,
+      if (!endpoint) {
+        throw new Error("Endpoint not found");
       }
-    );
+
+      if (endpoint.indexingHookIdentifier !== indexHookIdentifier) {
+        throw new Error("Index hook identifier is invalid");
+      }
+
+      const reason = parseReasonFromBody(body);
+
+      const index = await tx.endpointIndex.create({
+        data: {
+          endpointId: endpoint.id,
+          status: "PENDING",
+          source: "HOOK",
+          reason,
+          sourceData: body,
+        },
+      });
+
+      // Index the endpoint in 5 seconds from now
+      await workerQueue.enqueue(
+        "performEndpointIndexing",
+        {
+          id: index.id,
+        },
+        {
+          runAt: new Date(Date.now() + 5000),
+          maxAttempts:
+            endpoint.environment.type === RuntimeEnvironmentType.DEVELOPMENT ? 1 : undefined,
+          tx,
+        }
+      );
+    });
   }
 }
 

apps/webapp/app/routes/api.v1.runs.$runId.statuses.$id.ts

@@ -1,10 +1,7 @@
 import type { ActionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import { TaskStatus } from "@trigger.dev/database";
 import {
-  RunTaskBodyOutput,
-  RunTaskBodyOutputSchema,
-  ServerTask,
+  JobRunStatusRecordSchema,
   StatusHistory,
   StatusHistorySchema,
   StatusUpdate,
@@ -14,12 +11,8 @@ import {
 } from "@trigger.dev/core";
 import { z } from "zod";
 import { $transaction, PrismaClient, prisma } from "~/db.server";
-import { taskWithAttemptsToServerTask } from "~/models/task.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { ulid } from "~/services/ulid.server";
-import { workerQueue } from "~/services/worker.server";
-import { JobRunStatusRecordSchema } from "@trigger.dev/core";
 
 const ParamsSchema = z.object({
   runId: z.string(),

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete.ts

@@ -3,7 +3,7 @@ import { json } from "@remix-run/server-runtime";
 import type { CompleteTaskBodyOutput, ServerTask } from "@trigger.dev/core";
 import { CompleteTaskBodyInputSchema } from "@trigger.dev/core";
 import { z } from "zod";
-import { PrismaClient, prisma } from "~/db.server";
+import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
 import { taskWithAttemptsToServerTask } from "~/models/task.server";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
@@ -72,9 +72,9 @@ export async function action({ request, params }: ActionArgs) {
 }
 
 export class CompleteRunTaskService {
-  #prismaClient: PrismaClient;
+  #prismaClient: PrismaClientOrTransaction;
 
-  constructor(prismaClient: PrismaClient = prisma) {
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
     this.#prismaClient = prismaClient;
   }
 
@@ -86,7 +86,7 @@ export class CompleteRunTaskService {
   ): Promise<ServerTask | undefined> {
     // Using a transaction, we'll first check to see if the task already exists and return if if it does
     // If it doesn't exist, we'll create it and return it
-    const task = await this.#prismaClient.$transaction(async (tx) => {
+    const task = await $transaction(this.#prismaClient, async (tx) => {
       const existingTask = await tx.task.findUnique({
         where: {
           id,
@@ -152,6 +152,7 @@ export class CompleteRunTaskService {
         },
         include: {
           attempts: true,
+          run: true,
         },
       });
     });

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.fail.ts

@@ -152,6 +152,7 @@ export class FailRunTaskService {
         },
         include: {
           attempts: true,
+          run: true,
         },
       });
     });

apps/webapp/app/routes/api.v1.runs.$runId.tasks.ts

@@ -184,6 +184,7 @@ export class RunTaskService {
         },
         include: {
           attempts: true,
+          run: true,
         },
       });
 
@@ -280,7 +281,7 @@ export class RunTaskService {
           noop: taskBody.noop,
           delayUntil: taskBody.delayUntil,
           params: taskBody.params ?? undefined,
-          properties: taskBody.properties ?? undefined,
+          properties: this.#filterProperties(taskBody.properties) ?? undefined,
           redact: taskBody.redact ?? undefined,
           operation: taskBody.operation,
           callbackUrl,
@@ -305,7 +306,7 @@ export class RunTaskService {
           {
             id: task.id,
           },
-          { tx, runAt: task.delayUntil ?? undefined }
+          { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
         );
       } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
         if (taskBody.callback.timeoutInSeconds > 0) {
@@ -325,4 +326,14 @@ export class RunTaskService {
 
     return task ? taskWithAttemptsToServerTask(task) : undefined;
   }
+
+  #filterProperties(properties: RunTaskBodyOutput["properties"]): RunTaskBodyOutput["properties"] {
+    if (!properties) return;
+
+    return properties.filter((property) => {
+      if (!property) return false;
+
+      return typeof property.label === "string" && typeof property.text === "string";
+    });
+  }
 }

apps/webapp/app/routes/api.v1.sources.http.$id.ts

@@ -6,11 +6,24 @@ import { HandleHttpSourceService } from "~/services/sources/handleHttpSource.ser
 export async function action({ request, params }: ActionArgs) {
   logger.info("Handling http source", { url: request.url });
 
-  const { id } = z.object({ id: z.string() }).parse(params);
+  try {
+    const { id } = z.object({ id: z.string() }).parse(params);
+    const service = new HandleHttpSourceService();
+    const result = await service.call(id, request);
 
-  const service = new HandleHttpSourceService();
-
-  return await service.call(id, request);
+    return new Response(undefined, {
+      status: result.status,
+    });
+  } catch (e) {
+    if (e instanceof Error) {
+      logger.error("Error handling http source", { error: e.message });
+    } else {
+      logger.error("Error handling http source", { error: JSON.stringify(e) });
+    }
+    return new Response(undefined, {
+      status: 500,
+    });
+  }
 }
 
 export async function loader({ request, params }: LoaderArgs) {

apps/webapp/app/routes/login._index/route.tsx

@@ -60,16 +60,29 @@ export default function LoginPage() {
             <a href="https://trigger.dev">
               <LogoIcon className="mb-4 h-16 w-16" />
             </a>
-            <FormTitle divide={false} title="Log in to Trigger.dev" />
+            <FormTitle divide={false} title="Welcome to Trigger.dev" className="mb-2 pb-0" />
+            <Paragraph variant="small" className="mb-4">
+              Create an account or login
+            </Paragraph>
             <Fieldset>
               <div className="flex flex-col gap-y-2">
                 {data.showGithubAuth && (
-                  <Button type="submit" variant="primary/large" fullWidth>
+                  <Button
+                    type="submit"
+                    variant="primary/large"
+                    fullWidth
+                    data-action="continue with github"
+                  >
                     <NamedIcon name={"github"} className={"mr-1.5 h-4 w-4"} />
                     Continue with GitHub
                   </Button>
                 )}
-                <LinkButton to="/login/magic" variant="secondary/large" fullWidth>
+                <LinkButton
+                  to="/login/magic"
+                  variant="secondary/large"
+                  fullWidth
+                  data-action="continue with email"
+                >
                   <NamedIcon
                     name={"envelope"}
                     className={"mr-1.5 h-4 w-4 text-dimmed transition group-hover:text-bright"}

apps/webapp/app/routes/login.magic/route.tsx

@@ -1,6 +1,6 @@
 import type { ActionArgs, LoaderArgs } from "@remix-run/node";
 import { redirect } from "@remix-run/node";
-import { Form, useTransition } from "@remix-run/react";
+import { Form, useNavigation, useTransition } from "@remix-run/react";
 import { TypedMetaFunction, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { LogoIcon } from "~/components/LogoIcon";
@@ -17,10 +17,10 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { authenticator } from "~/services/auth.server";
 import { commitSession, getUserSession } from "~/services/sessionStorage.server";
 import magicLinkIcon from "./login.magic.svg";
-
 import type { LoaderType as RootLoader } from "~/root";
 import { appEnvTitleTag } from "~/utils";
 import { TextLink } from "~/components/primitives/TextLink";
+import { FormError } from "~/components/primitives/FormError";
 
 export const meta: TypedMetaFunction<typeof loader, { root: RootLoader }> = ({ parentsData }) => ({
   title: `Login to Trigger.dev${appEnvTitleTag(parentsData?.root.appEnv)}`,
@@ -32,10 +32,26 @@ export async function loader({ request }: LoaderArgs) {
   });
 
   const session = await getUserSession(request);
+  const error = session.get("auth:error");
 
-  return typedjson({
-    magicLinkSent: session.has("triggerdotdev:magiclink"),
-  });
+  let magicLinkError: string | undefined;
+  if (error) {
+    if ("message" in error) {
+      magicLinkError = error.message;
+    } else {
+      magicLinkError = JSON.stringify(error, null, 2);
+    }
+  }
+
+  return typedjson(
+    {
+      magicLinkSent: session.has("triggerdotdev:magiclink"),
+      magicLinkError,
+    },
+    {
+      headers: { "Set-Cookie": await commitSession(session) },
+    }
+  );
 }
 
 export async function action({ request }: ActionArgs) {
@@ -50,7 +66,7 @@ export async function action({ request }: ActionArgs) {
     .parse(payload);
 
   if (action === "send") {
-    await authenticator.authenticate("email-link", request, {
+    return authenticator.authenticate("email-link", request, {
       successRedirect: "/login/magic",
       failureRedirect: "/login/magic",
     });
@@ -67,13 +83,13 @@ export async function action({ request }: ActionArgs) {
 }
 
 export default function LoginMagicLinkPage() {
-  const { magicLinkSent } = useTypedLoaderData<typeof loader>();
-  const transition = useTransition();
+  const { magicLinkSent, magicLinkError } = useTypedLoaderData<typeof loader>();
+  const navigate = useNavigation();
 
   const isLoading =
-    (transition.state === "loading" || transition.state === "submitting") &&
-    transition.type === "actionSubmission" &&
-    transition.submission.formData.get("action") === "send";
+    (navigate.state === "loading" || navigate.state === "submitting") &&
+    navigate.formAction !== undefined &&
+    navigate.formData?.get("action") === "send";
 
   return (
     <AppContainer showBackgroundGradient={true}>
@@ -102,12 +118,17 @@ export default function LoginMagicLinkPage() {
                         variant="tertiary/small"
                         LeadingIcon="arrow-left"
                         leadingIconClassName="text-dimmed group-hover:text-bright transition"
+                        data-action="re-enter email"
                       >
                         Re-enter email
                       </Button>
                     }
                     confirmButton={
-                      <LinkButton to="/login" variant="tertiary/small">
+                      <LinkButton
+                        to="/login"
+                        variant="tertiary/small"
+                        data-action="log in using another option"
+                      >
                         Log in using another option
                       </LinkButton>
                     }
@@ -116,7 +137,10 @@ export default function LoginMagicLinkPage() {
               </>
             ) : (
               <>
-                <FormTitle divide={false} title="Log in to Trigger.dev" />
+                <FormTitle divide={false} title="Welcome to Trigger.dev" className="mb-2 pb-0" />
+                <Paragraph variant="small" className="mb-4 text-center">
+                  Create an account or login using your email
+                </Paragraph>
                 <Fieldset className="flex w-full flex-col items-center gap-y-2">
                   <InputGroup>
                     <Label>Your email address</Label>
@@ -137,6 +161,7 @@ export default function LoginMagicLinkPage() {
                     variant="primary/medium"
                     disabled={isLoading}
                     fullWidth
+                    data-action="send a magic link"
                   >
                     <NamedIcon
                       name={isLoading ? "spinner-white" : "envelope"}
@@ -144,6 +169,7 @@ export default function LoginMagicLinkPage() {
                     />
                     {isLoading ? "Sending…" : "Send a magic link"}
                   </Button>
+                  {magicLinkError && <FormError>{magicLinkError}</FormError>}
                 </Fieldset>
                 <Paragraph variant="extra-small" className="my-4 text-center">
                   By logging in with your email you agree to our{" "}
@@ -162,11 +188,28 @@ export default function LoginMagicLinkPage() {
                   variant={"tertiary/small"}
                   LeadingIcon={"arrow-left"}
                   leadingIconClassName="text-dimmed group-hover:text-bright transition"
+                  data-action="all login options"
                 >
                   All login options
                 </LinkButton>
               </>
             )}
+            <div className="mt-8 rounded border border-border px-6 py-4">
+              <Paragraph variant="small" className="mb-2 text-center">
+                Having login issues?
+              </Paragraph>
+              <Paragraph variant="extra-small" className="text-center">
+                Ensure the Magic Link email isn't in your spam folder. If the problem persists,{" "}
+                <TextLink href="mailto:help@trigger.dev" target="_blank">
+                  drop us an email
+                </TextLink>{" "}
+                or let us know on{" "}
+                <TextLink href="https://trigger.dev/discord" target="_blank">
+                  Discord
+                </TextLink>
+                .
+              </Paragraph>
+            </div>
           </div>
         </Form>
       </MainCenteredContainer>

apps/webapp/app/routes/resources.environments.$environmentParam.endpoint.$endpointParam.ts

@@ -1,26 +1,26 @@
-import { parse } from "@conform-to/zod";
 import { ActionArgs, json } from "@remix-run/server-runtime";
 import { z } from "zod";
-import { prisma } from "~/db.server";
-import {
-  CreateEndpointError,
-  CreateEndpointService,
-} from "~/services/endpoints/createEndpoint.server";
 import { IndexEndpointService } from "~/services/endpoints/indexEndpoint.server";
-import { requireUserId } from "~/services/session.server";
+import { workerQueue } from "~/services/worker.server";
 
 const ParamsSchema = z.object({
   environmentParam: z.string(),
   endpointParam: z.string(),
 });
 
-export async function action({ request, params }: ActionArgs) {
-  const userId = await requireUserId(request);
-  const { environmentParam, endpointParam } = ParamsSchema.parse(params);
+export async function action({ params }: ActionArgs) {
+  const { endpointParam } = ParamsSchema.parse(params);
 
   try {
     const service = new IndexEndpointService();
-    const result = await service.call(endpointParam, "MANUAL");
+    await service.call(endpointParam, "MANUAL");
+
+    // Enqueue the endpoint to be probed in 10 seconds
+    await workerQueue.enqueue(
+      "probeEndpoint",
+      { id: endpointParam },
+      { jobKey: `probe:${endpointParam}`, runAt: new Date(Date.now() + 10000) }
+    );
 
     return json({ success: true });
   } catch (e) {

apps/webapp/app/services/email.server.ts

@@ -1,4 +1,4 @@
-import type { DeliverEmail } from "emails";
+import type { DeliverEmail, SendPlainTextOptions } from "emails";
 import { EmailClient } from "emails";
 import type { SendEmailOptions } from "remix-auth-email-link";
 import { redirect } from "remix-typedjson";
@@ -6,6 +6,7 @@ import { env } from "~/env.server";
 import type { User } from "~/models/user.server";
 import type { AuthUser } from "./authUser";
 import { workerQueue } from "./worker.server";
+import { logger } from "./logger.server";
 
 const client = new EmailClient({
   apikey: env.RESEND_API_KEY,
@@ -20,11 +21,22 @@ export async function sendMagicLinkEmail(options: SendEmailOptions<AuthUser>): P
     throw redirect(options.magicLink);
   }
 
-  return client.send({
-    email: "magic_link",
-    to: options.emailAddress,
-    magicLink: options.magicLink,
-  });
+  logger.debug("Sending magic link email", { emailAddress: options.emailAddress });
+
+  try {
+    return await client.send({
+      email: "magic_link",
+      to: options.emailAddress,
+      magicLink: options.magicLink,
+    });
+  } catch (error) {
+    logger.error("Error sending magic link email", { error: JSON.stringify(error) });
+    throw error;
+  }
+}
+
+export async function sendPlainTextEmail(options: SendPlainTextOptions) {
+  return client.sendPlainText(options);
 }
 
 export async function scheduleWelcomeEmail(user: User) {

apps/webapp/app/services/emailAuth.server.tsx

@@ -5,6 +5,7 @@ import { findOrCreateUser } from "~/models/user.server";
 import { env } from "~/env.server";
 import { sendMagicLinkEmail } from "~/services/email.server";
 import { postAuthentication } from "./postAuth.server";
+import { logger } from "./logger.server";
 
 let secret = env.MAGIC_LINK_SECRET;
 if (!secret) throw new Error("Missing MAGIC_LINK_SECRET env variable.");
@@ -25,6 +26,8 @@ const emailStrategy = new EmailLinkStrategy(
     form: FormData;
     magicLinkVerify: boolean;
   }) => {
+    logger.info("Magic link user authenticated", { email, magicLinkVerify });
+
     try {
       const { user, isNewUser } = await findOrCreateUser({
         email,
@@ -35,6 +38,7 @@ const emailStrategy = new EmailLinkStrategy(
 
       return { userId: user.id };
     } catch (error) {
+      logger.debug("Magic link user failed to authenticate", { error: JSON.stringify(error) });
       throw error;
     }
   }

apps/webapp/app/services/endpointApi.server.ts

@@ -97,6 +97,7 @@ export class EndpointApi {
       return {
         ...pongResponse.data,
         triggerVersion: headers.data["trigger-version"],
+        triggerSdkVersion: headers.data["trigger-sdk-version"],
       };
     }
 
@@ -104,6 +105,7 @@ export class EndpointApi {
   }
 
   async indexEndpoint() {
+    const startTimeInMs = performance.now();
     const response = await safeFetch(this.url, {
       method: "POST",
       headers: {
@@ -113,40 +115,13 @@ export class EndpointApi {
       },
     });
 
-    if (!response) {
-      throw new Error(`Could not connect to endpoint ${this.url}`);
-    }
-
-    if (response.status === 401) {
-      const body = await safeBodyFromResponse(response, ErrorWithStackSchema);
-
-      if (body) {
-        return {
-          ok: false,
-          error: body.message,
-        } as const;
-      }
-
-      return {
-        ok: false,
-        error: `Trigger API key is invalid`,
-      } as const;
-    }
-
-    if (!response.ok) {
-      throw new Error(`Could not connect to endpoint ${this.url}. Status code: ${response.status}`);
-    }
-
-    const anyBody = await response.json();
-
-    const data = IndexEndpointResponseSchema.parse(anyBody);
-    const headers = EndpointHeadersSchema.parse(Object.fromEntries(response.headers.entries()));
-
     return {
-      ok: true,
-      data,
-      headers,
-    } as const;
+      response,
+      headerParser: EndpointHeadersSchema,
+      parser: IndexEndpointResponseSchema,
+      errorParser: ErrorWithStackSchema,
+      durationInMs: Math.floor(performance.now() - startTimeInMs),
+    };
   }
 
   async executeJobRequest(options: RunJobBody) {
@@ -334,6 +309,27 @@ export class EndpointApi {
 
     return validateResponse.data;
   }
+
+  async probe(timeout: number) {
+    const startTimeInMs = performance.now();
+
+    const response = await safeFetch(this.url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "x-trigger-api-key": this.apiKey,
+        "x-trigger-action": "PROBE_EXECUTION_TIMEOUT",
+      },
+      body: JSON.stringify({
+        timeout,
+      }),
+    });
+
+    return {
+      response,
+      durationInMs: Math.floor(performance.now() - startTimeInMs),
+    };
+  }
 }
 
 async function safeFetch(url: string, options: RequestInit) {

apps/webapp/app/services/endpoints/createEndpoint.server.ts

@@ -82,12 +82,19 @@ export class CreateEndpointService {
           },
         });
 
+        const endpointIndex = await tx.endpointIndex.create({
+          data: {
+            endpointId: endpoint.id,
+            status: "PENDING",
+            source: "INTERNAL",
+          },
+        });
+
         // Kick off process to fetch the jobs for this endpoint
         await workerQueue.enqueue(
-          "indexEndpoint",
+          "performEndpointIndexing",
           {
-            id: endpoint.id,
-            source: "INTERNAL",
+            id: endpointIndex.id,
           },
           {
             tx,
@@ -96,7 +103,7 @@ export class CreateEndpointService {
           }
         );
 
-        return endpoint;
+        return { ...endpoint, endpointIndex };
       });
 
       return result;

apps/webapp/app/services/endpoints/indexEndpoint.server.ts

@@ -1,23 +1,9 @@
 import type { EndpointIndexSource } from "@trigger.dev/database";
 import { PrismaClient, prisma } from "~/db.server";
-import { findEndpoint } from "~/models/endpoint.server";
-import { EndpointApi } from "../endpointApi.server";
-import { RegisterJobService } from "../jobs/registerJob.server";
-import { logger } from "../logger.server";
-import { RegisterSourceServiceV1 } from "../sources/registerSourceV1.server";
-import { RegisterDynamicScheduleService } from "../triggers/registerDynamicSchedule.server";
-import { RegisterDynamicTriggerService } from "../triggers/registerDynamicTrigger.server";
-import { DisableJobService } from "../jobs/disableJob.server";
-import { RegisterSourceServiceV2 } from "../sources/registerSourceV2.server";
+import { PerformEndpointIndexService } from "./performEndpointIndexService";
 
 export class IndexEndpointService {
   #prismaClient: PrismaClient;
-  #registerJobService = new RegisterJobService();
-  #disableJobService = new DisableJobService();
-  #registerSourceServiceV1 = new RegisterSourceServiceV1();
-  #registerSourceServiceV2 = new RegisterSourceServiceV2();
-  #registerDynamicTriggerService = new RegisterDynamicTriggerService();
-  #registerDynamicScheduleService = new RegisterDynamicScheduleService();
 
   constructor(prismaClient: PrismaClient = prisma) {
     this.#prismaClient = prismaClient;
@@ -29,220 +15,17 @@ export class IndexEndpointService {
     reason?: string,
     sourceData?: any
   ) {
-    const endpoint = await findEndpoint(id);
-
-    // Make a request to the endpoint to fetch a list of jobs
-    const client = new EndpointApi(endpoint.environment.apiKey, endpoint.url);
-
-    const indexResponse = await client.indexEndpoint();
-
-    if (!indexResponse.ok) {
-      throw new Error(indexResponse.error);
-    }
-
-    const { jobs, sources, dynamicTriggers, dynamicSchedules } = indexResponse.data;
-    const { "trigger-version": triggerVersion } = indexResponse.headers;
-
-    logger.debug("Indexing endpoint", {
-      endpointId: endpoint.id,
-      endpointUrl: endpoint.url,
-      endpointSlug: endpoint.slug,
-      source: source,
-      sourceData: sourceData,
-      triggerVersion,
-      stats: {
-        jobs: jobs.length,
-        sources: sources.length,
-        dynamicTriggers: dynamicTriggers.length,
-        dynamicSchedules: dynamicSchedules.length,
-      },
-    });
-
-    if (triggerVersion && triggerVersion !== endpoint.version) {
-      await this.#prismaClient.endpoint.update({
-        where: {
-          id: endpoint.id,
-        },
-        data: {
-          version: triggerVersion,
-        },
-      });
-    }
-
-    const indexStats = {
-      jobs: 0,
-      sources: 0,
-      dynamicTriggers: 0,
-      dynamicSchedules: 0,
-      disabledJobs: 0,
-    };
-
-    const existingJobs = await this.#prismaClient.job.findMany({
-      where: {
-        projectId: endpoint.projectId,
-        deletedAt: null,
-      },
-      include: {
-        aliases: {
-          where: {
-            name: "latest",
-            environmentId: endpoint.environmentId,
-          },
-          include: {
-            version: true,
-          },
-          take: 1,
-        },
-      },
-    });
-
-    for (const job of jobs) {
-      if (!job.enabled) {
-        const disabledJob = await this.#disableJobService
-          .call(endpoint, { slug: job.id, version: job.version })
-          .catch((error) => {
-            logger.error("Failed to disable job", {
-              endpointId: endpoint.id,
-              job,
-              error,
-            });
-
-            return;
-          });
-
-        if (disabledJob) {
-          indexStats.disabledJobs++;
-        }
-      } else {
-        try {
-          const registeredVersion = await this.#registerJobService.call(endpoint, job);
-
-          if (registeredVersion) {
-            indexStats.jobs++;
-          }
-        } catch (error) {
-          logger.error("Failed to register job", {
-            endpointId: endpoint.id,
-            job,
-            error,
-          });
-        }
-      }
-    }
-
-    // TODO: we need to do this for sources, dynamic triggers, and dynamic schedules
-    const missingJobs = existingJobs.filter((job) => {
-      return !jobs.find((j) => j.id === job.slug);
-    });
-
-    if (missingJobs.length > 0) {
-      logger.debug("Disabling missing jobs", {
-        endpointId: endpoint.id,
-        missingJobIds: missingJobs.map((job) => job.slug),
-      });
-
-      for (const job of missingJobs) {
-        const latestVersion = job.aliases[0]?.version;
-
-        if (!latestVersion) {
-          continue;
-        }
-
-        const disabledJob = await this.#disableJobService
-          .call(endpoint, {
-            slug: job.slug,
-            version: latestVersion.version,
-          })
-          .catch((error) => {
-            logger.error("Failed to disable job", {
-              endpointId: endpoint.id,
-              job,
-              error,
-            });
-
-            return;
-          });
-
-        if (disabledJob) {
-          indexStats.disabledJobs++;
-        }
-      }
-    }
-
-    for (const source of sources) {
-      try {
-        switch (source.version) {
-          default:
-          case "1": {
-            await this.#registerSourceServiceV1.call(endpoint, source);
-            break;
-          }
-          case "2": {
-            await this.#registerSourceServiceV2.call(endpoint, source);
-            break;
-          }
-        }
-
-        indexStats.sources++;
-      } catch (error) {
-        logger.error("Failed to register source", {
-          endpointId: endpoint.id,
-          source,
-          error,
-        });
-      }
-    }
-
-    for (const dynamicTrigger of dynamicTriggers) {
-      try {
-        await this.#registerDynamicTriggerService.call(endpoint, dynamicTrigger);
-
-        indexStats.dynamicTriggers++;
-      } catch (error) {
-        logger.error("Failed to register dynamic trigger", {
-          endpointId: endpoint.id,
-          dynamicTrigger,
-          error,
-        });
-      }
-    }
-
-    for (const dynamicSchedule of dynamicSchedules) {
-      try {
-        await this.#registerDynamicScheduleService.call(endpoint, dynamicSchedule);
-
-        indexStats.dynamicSchedules++;
-      } catch (error) {
-        logger.error("Failed to register dynamic schedule", {
-          endpointId: endpoint.id,
-          dynamicSchedule,
-          error,
-        });
-      }
-    }
-
-    logger.debug("Endpoint indexing complete", {
-      endpointId: endpoint.id,
-      indexStats,
-      source,
-      sourceData,
-      reason,
-    });
-
-    return await this.#prismaClient.endpointIndex.create({
+    const endpointIndex = await this.#prismaClient.endpointIndex.create({
       data: {
-        endpointId: endpoint.id,
-        stats: indexStats,
-        data: {
-          jobs,
-          sources,
-          dynamicTriggers,
-          dynamicSchedules,
-        },
+        endpointId: id,
+        status: "PENDING",
         source,
-        sourceData,
         reason,
+        sourceData,
       },
     });
+
+    const performEndpointIndexService = new PerformEndpointIndexService();
+    return await performEndpointIndexService.call(endpointIndex.id);
   }
 }

apps/webapp/app/services/endpoints/performEndpointIndexService.ts

@@ -0,0 +1,343 @@
+import type { EndpointIndexSource } from "@trigger.dev/database";
+import { PrismaClient, prisma } from "~/db.server";
+import { findEndpoint } from "~/models/endpoint.server";
+import { EndpointApi } from "../endpointApi.server";
+import { RegisterJobService } from "../jobs/registerJob.server";
+import { logger } from "../logger.server";
+import { RegisterSourceServiceV1 } from "../sources/registerSourceV1.server";
+import { RegisterDynamicScheduleService } from "../triggers/registerDynamicSchedule.server";
+import { RegisterDynamicTriggerService } from "../triggers/registerDynamicTrigger.server";
+import { DisableJobService } from "../jobs/disableJob.server";
+import { RegisterSourceServiceV2 } from "../sources/registerSourceV2.server";
+import { EndpointIndexError } from "@trigger.dev/core";
+import { safeBodyFromResponse } from "~/utils/json";
+import { fromZodError } from "zod-validation-error";
+import { IndexEndpointStats } from "@trigger.dev/core";
+
+export class PerformEndpointIndexService {
+  #prismaClient: PrismaClient;
+  #registerJobService = new RegisterJobService();
+  #disableJobService = new DisableJobService();
+  #registerSourceServiceV1 = new RegisterSourceServiceV1();
+  #registerSourceServiceV2 = new RegisterSourceServiceV2();
+  #registerDynamicTriggerService = new RegisterDynamicTriggerService();
+  #registerDynamicScheduleService = new RegisterDynamicScheduleService();
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const endpointIndex = await this.#prismaClient.endpointIndex.update({
+      where: {
+        id,
+      },
+      data: {
+        status: "STARTED",
+      },
+      include: {
+        endpoint: {
+          include: {
+            environment: {
+              include: {
+                organization: true,
+                project: true,
+              },
+            },
+          },
+        },
+      },
+    });
+
+    logger.debug("Performing endpoint index", endpointIndex);
+
+    // Make a request to the endpoint to fetch a list of jobs
+    const client = new EndpointApi(
+      endpointIndex.endpoint.environment.apiKey,
+      endpointIndex.endpoint.url
+    );
+    const { response, parser, headerParser, errorParser } = await client.indexEndpoint();
+
+    if (!response) {
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
+      });
+    }
+
+    if (response.status === 401) {
+      const body = await safeBodyFromResponse(response, errorParser);
+
+      if (body) {
+        return updateEndpointIndexWithError(this.#prismaClient, id, {
+          message: body.message,
+        });
+      }
+
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: "Trigger API key is invalid",
+      });
+    }
+
+    if (!response.ok) {
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
+      });
+    }
+
+    const anyBody = await response.json();
+    const bodyResult = parser.safeParse(anyBody);
+
+    if (!bodyResult.success) {
+      const issues: string[] = [];
+      bodyResult.error.issues.forEach((issue) => {
+        if (issue.path.at(0) === "jobs") {
+          const jobIndex = issue.path.at(1) as number;
+          const job = (anyBody as any).jobs[jobIndex];
+
+          if (job) {
+            issues.push(`Job "${job.id}": ${issue.message} at "${issue.path.slice(2).join(".")}".`);
+          }
+        }
+      });
+
+      let friendlyError: string | undefined;
+      if (issues.length > 0) {
+        friendlyError = `Your Jobs have issues:\n${issues.map((issue) => `- ${issue}`).join("\n")}`;
+      } else {
+        friendlyError = fromZodError(bodyResult.error, {
+          prefix: "There's an issue with the format of your Jobs",
+        }).message;
+      }
+
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: friendlyError,
+        raw: bodyResult.error.issues,
+      });
+    }
+
+    const headerResult = headerParser.safeParse(Object.fromEntries(response.headers.entries()));
+    if (!headerResult.success) {
+      const friendlyError = fromZodError(headerResult.error, {
+        prefix: "Your headers are invalid",
+      });
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: friendlyError.message,
+        raw: headerResult.error.issues,
+      });
+    }
+
+    const { jobs, sources, dynamicTriggers, dynamicSchedules } = bodyResult.data;
+    const { "trigger-version": triggerVersion, "trigger-sdk-version": triggerSdkVersion } =
+      headerResult.data;
+    const { endpoint } = endpointIndex;
+
+    if (
+      (triggerVersion && triggerVersion !== endpoint.version) ||
+      (triggerSdkVersion && triggerSdkVersion !== endpoint.sdkVersion)
+    ) {
+      await this.#prismaClient.endpoint.update({
+        where: {
+          id: endpoint.id,
+        },
+        data: {
+          version: triggerVersion,
+          sdkVersion: triggerSdkVersion,
+        },
+      });
+    }
+
+    const indexStats: IndexEndpointStats = {
+      jobs: 0,
+      sources: 0,
+      dynamicTriggers: 0,
+      dynamicSchedules: 0,
+      disabledJobs: 0,
+    };
+
+    const existingJobs = await this.#prismaClient.job.findMany({
+      where: {
+        projectId: endpoint.projectId,
+        deletedAt: null,
+      },
+      include: {
+        aliases: {
+          where: {
+            name: "latest",
+            environmentId: endpoint.environmentId,
+          },
+          include: {
+            version: true,
+          },
+          take: 1,
+        },
+      },
+    });
+
+    for (const job of jobs) {
+      if (!job.enabled) {
+        const disabledJob = await this.#disableJobService
+          .call(endpoint, { slug: job.id, version: job.version })
+          .catch((error) => {
+            logger.error("Failed to disable job", {
+              endpointId: endpoint.id,
+              job,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledJob) {
+          indexStats.disabledJobs++;
+        }
+      } else {
+        try {
+          const registeredVersion = await this.#registerJobService.call(endpoint, job);
+
+          if (registeredVersion) {
+            if (!job.internal) {
+              indexStats.jobs++;
+            }
+          }
+        } catch (error) {
+          logger.error("Failed to register job", {
+            endpointId: endpoint.id,
+            job,
+            error,
+          });
+        }
+      }
+    }
+
+    // TODO: we need to do this for sources, dynamic triggers, and dynamic schedules
+    const missingJobs = existingJobs.filter((job) => {
+      return !jobs.find((j) => j.id === job.slug);
+    });
+
+    if (missingJobs.length > 0) {
+      logger.debug("Disabling missing jobs", {
+        endpointId: endpoint.id,
+        missingJobIds: missingJobs.map((job) => job.slug),
+      });
+
+      for (const job of missingJobs) {
+        const latestVersion = job.aliases[0]?.version;
+
+        if (!latestVersion) {
+          continue;
+        }
+
+        const disabledJob = await this.#disableJobService
+          .call(endpoint, {
+            slug: job.slug,
+            version: latestVersion.version,
+          })
+          .catch((error) => {
+            logger.error("Failed to disable job", {
+              endpointId: endpoint.id,
+              job,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledJob) {
+          indexStats.disabledJobs++;
+        }
+      }
+    }
+
+    for (const source of sources) {
+      try {
+        switch (source.version) {
+          default:
+          case "1": {
+            await this.#registerSourceServiceV1.call(endpoint, source);
+            break;
+          }
+          case "2": {
+            await this.#registerSourceServiceV2.call(endpoint, source);
+            break;
+          }
+        }
+
+        indexStats.sources++;
+      } catch (error) {
+        logger.error("Failed to register source", {
+          endpointId: endpoint.id,
+          source,
+          error,
+        });
+      }
+    }
+
+    for (const dynamicTrigger of dynamicTriggers) {
+      try {
+        await this.#registerDynamicTriggerService.call(endpoint, dynamicTrigger);
+
+        indexStats.dynamicTriggers++;
+      } catch (error) {
+        logger.error("Failed to register dynamic trigger", {
+          endpointId: endpoint.id,
+          dynamicTrigger,
+          error,
+        });
+      }
+    }
+
+    for (const dynamicSchedule of dynamicSchedules) {
+      try {
+        await this.#registerDynamicScheduleService.call(endpoint, dynamicSchedule);
+
+        indexStats.dynamicSchedules++;
+      } catch (error) {
+        logger.error("Failed to register dynamic schedule", {
+          endpointId: endpoint.id,
+          dynamicSchedule,
+          error,
+        });
+      }
+    }
+
+    logger.debug("Endpoint indexing complete", {
+      endpointId: endpoint.id,
+      indexStats,
+      source: endpointIndex.source,
+      sourceData: endpointIndex.sourceData,
+      reason: endpointIndex.reason,
+    });
+
+    return await this.#prismaClient.endpointIndex.update({
+      where: {
+        id,
+      },
+      data: {
+        status: "SUCCESS",
+        stats: indexStats,
+        data: {
+          jobs,
+          sources,
+          dynamicTriggers,
+          dynamicSchedules,
+        },
+      },
+    });
+  }
+}
+
+async function updateEndpointIndexWithError(
+  prismaClient: PrismaClient,
+  id: string,
+  error: EndpointIndexError
+) {
+  return await prismaClient.endpointIndex.update({
+    where: {
+      id,
+    },
+    data: {
+      status: "FAILURE",
+      error,
+    },
+  });
+}

apps/webapp/app/services/endpoints/probeEndpoint.server.ts

@@ -0,0 +1,64 @@
+import { MAX_RUN_CHUNK_EXECUTION_LIMIT, RESPONSE_TIMEOUT_STATUS_CODES } from "~/consts";
+import { prisma, PrismaClient } from "~/db.server";
+import { EndpointApi } from "../endpointApi.server";
+import { logger } from "../logger.server";
+import { detectResponseIsTimeout } from "~/models/endpoint.server";
+
+export class ProbeEndpointService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const endpoint = await this.#prismaClient.endpoint.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        environment: true,
+      },
+    });
+
+    if (!endpoint) {
+      return;
+    }
+
+    logger.debug(`Probing endpoint`, {
+      id,
+    });
+
+    const client = new EndpointApi(endpoint.environment.apiKey, endpoint.url);
+
+    const { response, durationInMs } = await client.probe(MAX_RUN_CHUNK_EXECUTION_LIMIT);
+
+    if (!response) {
+      return;
+    }
+
+    logger.debug(`Probing endpoint complete`, {
+      id,
+      durationInMs,
+      response: {
+        status: response.status,
+        headers: Object.fromEntries(response.headers.entries()),
+      },
+    });
+
+    // If the response is a 200, or it was a timeout, we can assume the endpoint is up and update the runChunkExecutionLimit
+    if (response.status === 200 || detectResponseIsTimeout(response)) {
+      await this.#prismaClient.endpoint.update({
+        where: {
+          id,
+        },
+        data: {
+          runChunkExecutionLimit: Math.min(
+            Math.max(durationInMs, 10000),
+            MAX_RUN_CHUNK_EXECUTION_LIMIT
+          ),
+        },
+      });
+    }
+  }
+}

apps/webapp/app/services/endpoints/recurringEndpointIndex.server.ts

@@ -17,7 +17,9 @@ export class RecurringEndpointIndexService {
     const endpoints = await this.#prismaClient.endpoint.findMany({
       where: {
         environment: {
-          type: RuntimeEnvironmentType.PRODUCTION,
+          type: {
+            in: [RuntimeEnvironmentType.PRODUCTION, RuntimeEnvironmentType.STAGING],
+          },
         },
         indexings: {
           none: {
@@ -32,12 +34,18 @@ export class RecurringEndpointIndexService {
     logger.debug("Found endpoints that haven't been indexed in the last 10 minutes", {
       count: endpoints.length,
     });
-
     // Enqueue each endpoint for indexing
     for (const endpoint of endpoints) {
-      await workerQueue.enqueue("indexEndpoint", {
-        id: endpoint.id,
-        source: "INTERNAL",
+      const index = await this.#prismaClient.endpointIndex.create({
+        data: {
+          endpointId: endpoint.id,
+          status: "PENDING",
+          source: "INTERNAL",
+        },
+      });
+
+      await workerQueue.enqueue("performEndpointIndexing", {
+        id: index.id,
       });
     }
   }

apps/webapp/app/services/endpoints/validateCreateEndpoint.server.ts

@@ -66,12 +66,15 @@ export class ValidateCreateEndpointService {
           },
         });
 
-        // Kick off process to fetch the jobs for this endpoint
+        const index = await tx.endpointIndex.create({
+          data: { endpointId: endpoint.id, status: "PENDING", source: "INTERNAL" },
+        });
+
+        // Kick off process to fetch the jobs for this index
         await workerQueue.enqueue(
-          "indexEndpoint",
+          "performEndpointIndexing",
           {
-            id: endpoint.id,
-            source: "INTERNAL",
+            id: index.id,
           },
           {
             tx,

apps/webapp/app/services/events/ingestSendEvent.server.ts

@@ -3,6 +3,25 @@ import { $transaction, PrismaClientOrTransaction, PrismaErrorSchema, prisma } fr
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { workerQueue } from "~/services/worker.server";
 import { logger } from "../logger.server";
+import { EventRecord, ExternalAccount } from "@trigger.dev/database";
+
+type UpdateEventInput = {
+  tx: PrismaClientOrTransaction;
+  existingEventLog: EventRecord;
+  reqEvent: RawEvent;
+  deliverAt?: Date;
+};
+
+type CreateEventInput = {
+  tx: PrismaClientOrTransaction;
+  event: RawEvent;
+  environment: AuthenticatedEnvironment;
+  deliverAt?: Date;
+  sourceContext?: { id: string; metadata?: any };
+  externalAccount?: ExternalAccount;
+};
+
+const EVENT_UPDATE_THRESHOLD_WINDOW_IN_MSECS = 5 * 1000; // 5 seconds
 
 export class IngestSendEvent {
   #prismaClient: PrismaClientOrTransaction;
@@ -52,34 +71,25 @@ export class IngestSendEvent {
             })
           : undefined;
 
-        // Create a new event in the database
-        const eventLog = await tx.eventRecord.create({
-          data: {
-            organizationId: environment.organizationId,
-            projectId: environment.projectId,
-            environmentId: environment.id,
-            eventId: event.id,
-            name: event.name,
-            timestamp: event.timestamp ?? new Date(),
-            payload: event.payload ?? {},
-            context: event.context ?? {},
-            source: event.source ?? "trigger.dev",
-            sourceContext,
-            deliverAt: deliverAt,
-            externalAccountId: externalAccount ? externalAccount.id : undefined,
+        const existingEventLog = await tx.eventRecord.findUnique({
+          where: {
+            eventId_environmentId: {
+              eventId: event.id,
+              environmentId: environment.id,
+            },
           },
         });
 
-        if (this.deliverEvents) {
-          // Produce a message to the event bus
-          await workerQueue.enqueue(
-            "deliverEvent",
-            {
-              id: eventLog.id,
-            },
-            { runAt: eventLog.deliverAt, tx, jobKey: `event:${eventLog.id}` }
-          );
-        }
+        const eventLog = await (existingEventLog
+          ? this.updateEvent({ tx, existingEventLog, reqEvent: event, deliverAt })
+          : this.createEvent({
+              tx,
+              event,
+              environment,
+              deliverAt,
+              sourceContext,
+              externalAccount,
+            }));
 
         return eventLog;
       });
@@ -95,21 +105,81 @@ export class IngestSendEvent {
         throw error;
       }
 
-      // If the error is a Prisma unique constraint error, it means that the event already exists
-      if (prismaError.success && prismaError.data.code === "P2002") {
-        logger.debug("Event already exists, finding and returning", { event, environment });
-
-        return this.#prismaClient.eventRecord.findUniqueOrThrow({
-          where: {
-            eventId_environmentId: {
-              eventId: event.id,
-              environmentId: environment.id,
-            },
-          },
-        });
-      }
-
       throw error;
     }
   }
+
+  private async createEvent({
+    tx,
+    event,
+    environment,
+    deliverAt,
+    sourceContext,
+    externalAccount,
+  }: CreateEventInput) {
+    const eventLog = await tx.eventRecord.create({
+      data: {
+        organizationId: environment.organizationId,
+        projectId: environment.projectId,
+        environmentId: environment.id,
+        eventId: event.id,
+        name: event.name,
+        timestamp: event.timestamp ?? new Date(),
+        payload: event.payload ?? {},
+        context: event.context ?? {},
+        source: event.source ?? "trigger.dev",
+        sourceContext,
+        deliverAt: deliverAt,
+        externalAccountId: externalAccount ? externalAccount.id : undefined,
+      },
+    });
+
+    await this.enqueueWorkerEvent(tx, eventLog);
+
+    return eventLog;
+  }
+
+  private async updateEvent({ tx, existingEventLog, reqEvent, deliverAt }: UpdateEventInput) {
+    if (!this.shouldUpdateEvent(existingEventLog)) {
+      logger.debug(`not updating event for event id: ${existingEventLog.eventId}`);
+      return existingEventLog;
+    }
+
+    const updatedEventLog = await tx.eventRecord.update({
+      where: {
+        eventId_environmentId: {
+          eventId: existingEventLog.eventId,
+          environmentId: existingEventLog.environmentId,
+        },
+      },
+      data: {
+        payload: reqEvent.payload ?? existingEventLog.payload,
+        context: reqEvent.context ?? existingEventLog.context,
+        deliverAt: deliverAt ?? new Date(),
+      },
+    });
+
+    await this.enqueueWorkerEvent(tx, updatedEventLog);
+
+    return updatedEventLog;
+  }
+
+  private shouldUpdateEvent(eventLog: EventRecord) {
+    const thresholdTime = new Date(Date.now() + EVENT_UPDATE_THRESHOLD_WINDOW_IN_MSECS);
+
+    return eventLog.deliverAt >= thresholdTime;
+  }
+
+  private async enqueueWorkerEvent(tx: PrismaClientOrTransaction, eventLog: EventRecord) {
+    if (this.deliverEvents) {
+      // Produce a message to the event bus
+      await workerQueue.enqueue(
+        "deliverEvent",
+        {
+          id: eventLog.id,
+        },
+        { runAt: eventLog.deliverAt, tx, jobKey: `event:${eventLog.id}` }
+      );
+    }
+  }
 }

apps/webapp/app/services/jobs/registerJob.server.ts

@@ -4,14 +4,7 @@ import {
   SCHEDULED_EVENT,
   TriggerMetadata,
 } from "@trigger.dev/core";
-import type {
-  Endpoint,
-  Integration,
-  Job,
-  JobIntegration,
-  JobIntegrationPayload,
-  JobVersion,
-} from "@trigger.dev/database";
+import type { Endpoint, Integration, Job, JobIntegration, JobVersion } from "@trigger.dev/database";
 import { DEFAULT_MAX_CONCURRENT_RUNS } from "~/consts";
 import type { PrismaClient } from "~/db.server";
 import { prisma } from "~/db.server";
@@ -229,7 +222,7 @@ export class RegisterJobService {
           },
           update: {
             name: example.name,
-            icon: example.icon,
+            icon: example.icon ?? null,
             payload: example.payload,
           },
         });

apps/webapp/app/services/logger.server.ts

@@ -8,3 +8,10 @@ export const logger = new Logger(
   ["examples", "output", "connectionString", "payload"],
   sensitiveDataReplacer
 );
+
+export const workerLogger = new Logger(
+  "worker",
+  (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+  ["examples", "output", "connectionString"],
+  sensitiveDataReplacer
+);

apps/webapp/app/services/runs/createRun.server.ts

@@ -70,6 +70,7 @@ export class CreateRunService {
             ? eventRecord.externalAccountId
             : undefined,
           isTest: eventRecord.isTest,
+          internal: job.internal,
         },
       });
 

apps/webapp/app/services/runs/forceYieldCoordinator.server.ts

@@ -0,0 +1,47 @@
+import { PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { logger } from "../logger.server";
+
+class ForceYieldCoordinator {
+  private inFlightRuns: Set<string> = new Set();
+  private prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient) {
+    this.prismaClient = prismaClient;
+
+    process.on("SIGTERM", this.handleForceYield);
+  }
+
+  // Add a run to the in-flight set
+  public registerRun(runId: string): void {
+    this.inFlightRuns.add(runId);
+  }
+
+  // Remove a run from the in-flight set
+  public deregisterRun(runId: string): void {
+    this.inFlightRuns.delete(runId);
+  }
+
+  // Handle forced yield on SIGTERM
+  private handleForceYield = async (): Promise<void> => {
+    const runIds = Array.from(this.inFlightRuns);
+
+    const results = await this.prismaClient.jobRun.updateMany({
+      where: {
+        id: {
+          in: runIds,
+        },
+        forceYieldImmediately: false,
+      },
+      data: {
+        forceYieldImmediately: true,
+      },
+    });
+
+    logger.debug(
+      `ForceYieldCoordinator: ${results.count}/${runIds.length} runs set to immediately force yield`
+    );
+  };
+}
+
+export const forceYieldCoordinator = new ForceYieldCoordinator(prisma);

apps/webapp/app/services/runs/performRunExecutionV1.server.ts

@@ -1,771 +0,0 @@
-import {
-  CachedTaskSchema,
-  RunJobError,
-  RunJobInvalidPayloadError,
-  RunJobResumeWithTask,
-  RunJobRetryWithTask,
-  RunJobSuccess,
-  RunJobUnresolvedAuthError,
-  RunSourceContextSchema,
-} from "@trigger.dev/core";
-import type { Task } from "@trigger.dev/database";
-import { generateErrorMessage } from "zod-error";
-import { eventRecordToApiJson } from "~/api.server";
-import { EXECUTE_JOB_RETRY_LIMIT } from "~/consts";
-import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
-import { enqueueRunExecutionV1 } from "~/models/jobRunExecution.server";
-import { resolveRunConnections } from "~/models/runConnection.server";
-import { formatError } from "~/utils/formatErrors.server";
-import { safeJsonZodParse } from "~/utils/json";
-import { EndpointApi } from "../endpointApi.server";
-import { logger } from "../logger.server";
-
-type FoundRunExecution = NonNullable<Awaited<ReturnType<typeof findRunExecution>>>;
-
-export class PerformRunExecutionV1Service {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
-  public async call(id: string) {
-    const runExecution = await findRunExecution(this.#prismaClient, id);
-
-    if (!runExecution) {
-      return;
-    }
-
-    switch (runExecution.reason) {
-      case "PREPROCESS": {
-        await this.#executePreprocessing(runExecution);
-        break;
-      }
-      case "EXECUTE_JOB": {
-        await this.#executeJob(runExecution);
-        break;
-      }
-    }
-  }
-
-  // Execute the preprocessing step of a run, which will send the payload to the endpoint and give the job
-  // an opportunity to generate run properties based on the payload.
-  // If the endpoint is not available, or the response is not ok,
-  // the run execution will be marked as failed and the run will start
-  async #executePreprocessing(execution: FoundRunExecution) {
-    const { run } = execution;
-
-    const client = new EndpointApi(run.environment.apiKey, run.endpoint.url);
-    const event = eventRecordToApiJson(run.event);
-    const startedAt = new Date();
-
-    await this.#prismaClient.jobRunExecution.update({
-      where: {
-        id: execution.id,
-      },
-      data: {
-        status: "STARTED",
-        startedAt,
-      },
-    });
-
-    const { response, parser } = await client.preprocessRunRequest({
-      event,
-      job: {
-        id: run.version.job.slug,
-        version: run.version.version,
-      },
-      run: {
-        id: run.id,
-        isTest: run.isTest,
-      },
-      environment: {
-        id: run.environment.id,
-        slug: run.environment.slug,
-        type: run.environment.type,
-      },
-      organization: {
-        id: run.organization.id,
-        slug: run.organization.slug,
-        title: run.organization.title,
-      },
-      account: run.externalAccount
-        ? {
-            id: run.externalAccount.identifier,
-            metadata: run.externalAccount.metadata,
-          }
-        : undefined,
-    });
-
-    if (!response) {
-      return await this.#failRunExecutionWithRetry(execution, {
-        message: "Could not connect to the endpoint",
-      });
-    }
-
-    if (!response.ok) {
-      return await this.#failRunExecutionWithRetry(execution, {
-        message: `Endpoint responded with ${response.status} status code`,
-      });
-    }
-
-    const rawBody = await response.text();
-    const safeBody = safeJsonZodParse(parser, rawBody);
-
-    if (!safeBody) {
-      return await this.#failRunExecution(this.#prismaClient, execution, {
-        message: "Endpoint responded with invalid JSON",
-      });
-    }
-
-    if (!safeBody.success) {
-      return await this.#failRunExecution(this.#prismaClient, execution, {
-        message: generateErrorMessage(safeBody.error.issues),
-      });
-    }
-
-    if (safeBody.data.abort) {
-      return this.#failRunExecution(
-        this.#prismaClient,
-        execution,
-        { message: "Endpoint aborted the run" },
-        "ABORTED"
-      );
-    } else {
-      await $transaction(this.#prismaClient, async (tx) => {
-        await tx.jobRun.update({
-          where: {
-            id: run.id,
-          },
-          data: {
-            status: "STARTED",
-            startedAt: new Date(),
-            properties: safeBody.data.properties,
-          },
-        });
-
-        await tx.jobRunExecution.update({
-          where: {
-            id: execution.id,
-          },
-          data: {
-            status: "SUCCESS",
-            completedAt: new Date(),
-          },
-        });
-
-        const runExecution = await tx.jobRunExecution.create({
-          data: {
-            runId: run.id,
-            reason: "EXECUTE_JOB",
-            status: "PENDING",
-            retryLimit: EXECUTE_JOB_RETRY_LIMIT,
-          },
-        });
-
-        await enqueueRunExecutionV1(runExecution, run.queue.id, run.queue.maxJobs, tx);
-      });
-    }
-  }
-  async #executeJob(execution: FoundRunExecution) {
-    const { run, isRetry } = execution;
-
-    if (run.status === "CANCELED") {
-      await this.#cancelExecution(execution);
-      return;
-    }
-
-    const client = new EndpointApi(run.environment.apiKey, run.endpoint.url);
-    const event = eventRecordToApiJson(run.event);
-
-    const startedAt = new Date();
-
-    await this.#prismaClient.jobRunExecution.update({
-      where: {
-        id: execution.id,
-      },
-      data: {
-        status: "STARTED",
-        startedAt,
-        run: {
-          update: {
-            status: run.status === "QUEUED" ? "STARTED" : run.status,
-            startedAt: run.startedAt ?? new Date(),
-          },
-        },
-      },
-    });
-
-    const connections = await resolveRunConnections(run.runConnections);
-
-    if (!connections.success) {
-      return this.#failRunExecutionWithRetry(execution, {
-        message: `Could not resolve all connections for run ${run.id}, attempting to retry`,
-      });
-    }
-
-    let resumedTask: Task | undefined;
-
-    if (execution.resumeTaskId) {
-      resumedTask =
-        (await this.#prismaClient.task.findUnique({
-          where: {
-            id: execution.resumeTaskId,
-          },
-        })) ?? undefined;
-
-      if (resumedTask) {
-        resumedTask = await this.#prismaClient.task.update({
-          where: {
-            id: execution.resumeTaskId,
-          },
-          data: {
-            status: resumedTask.noop ? "COMPLETED" : "RUNNING",
-            completedAt: resumedTask.noop ? new Date() : undefined,
-          },
-        });
-      }
-    }
-
-    const sourceContext = RunSourceContextSchema.safeParse(run.event.sourceContext);
-
-    const { response, parser, errorParser } = await client.executeJobRequest({
-      event,
-      job: {
-        id: run.version.job.slug,
-        version: run.version.version,
-      },
-      run: {
-        id: run.id,
-        isTest: run.isTest,
-        startedAt,
-        isRetry,
-      },
-      environment: {
-        id: run.environment.id,
-        slug: run.environment.slug,
-        type: run.environment.type,
-      },
-      organization: {
-        id: run.organization.id,
-        slug: run.organization.slug,
-        title: run.organization.title,
-      },
-      account: run.externalAccount
-        ? {
-            id: run.externalAccount.identifier,
-            metadata: run.externalAccount.metadata,
-          }
-        : undefined,
-      connections: connections.auth,
-      source: sourceContext.success ? sourceContext.data : undefined,
-      tasks: [run.tasks, resumedTask]
-        .flat()
-        .filter(Boolean)
-        .map((t) => CachedTaskSchema.parse(t)),
-      yieldedExecutions: run.yieldedExecutions,
-    });
-
-    if (!response) {
-      return await this.#failRunExecutionWithRetry(execution, {
-        message: `Connection could not be established to the endpoint (${run.endpoint.url})`,
-      });
-    }
-
-    const rawBody = await response.text();
-
-    if (!response.ok) {
-      logger.debug("Endpoint responded with non-200 status code", {
-        status: response.status,
-        runId: run.id,
-        endpoint: run.endpoint.url,
-      });
-
-      const errorBody = safeJsonZodParse(errorParser, rawBody);
-
-      if (errorBody && errorBody.success) {
-        // Only retry if the error isn't a 4xx
-        if (response.status >= 400 && response.status <= 499) {
-          return await this.#failRunExecution(this.#prismaClient, execution, errorBody.data);
-        } else {
-          return await this.#failRunExecutionWithRetry(execution, errorBody.data);
-        }
-      }
-
-      // Only retry if the error isn't a 4xx
-      if (response.status >= 400 && response.status <= 499) {
-        return await this.#failRunExecution(this.#prismaClient, execution, {
-          message: `Endpoint responded with ${response.status} status code`,
-        });
-      } else {
-        return await this.#failRunExecutionWithRetry(execution, {
-          message: `Endpoint responded with ${response.status} status code`,
-        });
-      }
-    }
-
-    const safeBody = safeJsonZodParse(parser, rawBody);
-
-    if (!safeBody) {
-      return await this.#failRunExecution(this.#prismaClient, execution, {
-        message: "Endpoint responded with invalid JSON",
-      });
-    }
-
-    if (!safeBody.success) {
-      return await this.#failRunExecution(this.#prismaClient, execution, {
-        message: generateErrorMessage(safeBody.error.issues),
-      });
-    }
-
-    const status = safeBody.data.status;
-
-    switch (status) {
-      case "SUCCESS": {
-        await this.#completeRunWithSuccess(execution, safeBody.data);
-
-        break;
-      }
-      case "RESUME_WITH_TASK": {
-        await this.#resumeRunWithTask(execution, safeBody.data);
-
-        break;
-      }
-      case "ERROR": {
-        await this.#failRunWithError(execution, safeBody.data);
-
-        break;
-      }
-      case "RETRY_WITH_TASK": {
-        await this.#retryRunWithTask(execution, safeBody.data);
-
-        break;
-      }
-      case "CANCELED": {
-        await this.#cancelExecution(execution);
-        break;
-      }
-      case "UNRESOLVED_AUTH_ERROR": {
-        await this.#failRunWithUnresolvedAuthError(execution, safeBody.data);
-
-        break;
-      }
-      case "INVALID_PAYLOAD": {
-        await this.#failRunWithInvalidPayloadError(execution, safeBody.data);
-
-        break;
-      }
-      case "YIELD_EXECUTION": {
-        await this.#resumeYieldedExecution(execution, safeBody.data.key);
-
-        break;
-      }
-      default: {
-        const _exhaustiveCheck: never = status;
-        throw new Error(`Non-exhaustive match for value: ${status}`);
-      }
-    }
-  }
-
-  async #completeRunWithSuccess(execution: FoundRunExecution, data: RunJobSuccess) {
-    const { run } = execution;
-
-    return await $transaction(this.#prismaClient, async (tx) => {
-      await tx.jobRun.update({
-        where: { id: run.id },
-        data: {
-          completedAt: new Date(),
-          status: "SUCCESS",
-          output: data.output ?? undefined,
-          queue: {
-            update: {
-              jobCount: {
-                decrement: 1,
-              },
-            },
-          },
-        },
-      });
-
-      await tx.jobRunExecution.update({
-        where: {
-          id: execution.id,
-        },
-        data: {
-          status: "SUCCESS",
-          completedAt: new Date(),
-        },
-      });
-    });
-  }
-
-  async #resumeYieldedExecution(execution: FoundRunExecution, key: string) {
-    const { run } = execution;
-
-    return await $transaction(this.#prismaClient, async (tx) => {
-      await tx.jobRunExecution.update({
-        where: {
-          id: execution.id,
-        },
-        data: {
-          status: "SUCCESS",
-          completedAt: new Date(),
-          run: {
-            update: {
-              yieldedExecutions: {
-                push: key,
-              },
-            },
-          },
-        },
-      });
-
-      const newJobExecution = await tx.jobRunExecution.create({
-        data: {
-          runId: run.id,
-          reason: "EXECUTE_JOB",
-          status: "PENDING",
-          retryLimit: EXECUTE_JOB_RETRY_LIMIT,
-        },
-      });
-
-      await enqueueRunExecutionV1(newJobExecution, run.queue.id, run.queue.maxJobs, tx);
-    });
-  }
-
-  async #resumeRunWithTask(execution: FoundRunExecution, data: RunJobResumeWithTask) {
-    const { run } = execution;
-
-    return await $transaction(this.#prismaClient, async (tx) => {
-      await tx.jobRunExecution.update({
-        where: {
-          id: execution.id,
-        },
-        data: {
-          status: "SUCCESS",
-          completedAt: new Date(),
-        },
-      });
-
-      // If the task has an operation, then the next performRunExecution will occur
-      // when that operation has finished
-      // Tasks with callbacks enabled will also get processed separately, i.e. when
-      // they time out, or on valid requests to their callbackUrl
-      if (!data.task.operation && !data.task.callbackUrl) {
-        const newJobExecution = await tx.jobRunExecution.create({
-          data: {
-            runId: run.id,
-            reason: "EXECUTE_JOB",
-            status: "PENDING",
-            retryLimit: EXECUTE_JOB_RETRY_LIMIT,
-            resumeTaskId: data.task.id,
-          },
-        });
-
-        await enqueueRunExecutionV1(
-          newJobExecution,
-          run.queue.id,
-          run.queue.maxJobs,
-          tx,
-          data.task.delayUntil ?? undefined
-        );
-      }
-    });
-  }
-
-  async #failRunWithError(execution: FoundRunExecution, data: RunJobError) {
-    return await $transaction(this.#prismaClient, async (tx) => {
-      if (data.task) {
-        await tx.task.update({
-          where: {
-            id: data.task.id,
-          },
-          data: {
-            status: "ERRORED",
-            completedAt: new Date(),
-            output: data.error ?? undefined,
-          },
-        });
-      }
-
-      await this.#failRunExecution(tx, execution, data.error ?? undefined);
-    });
-  }
-
-  async #failRunWithUnresolvedAuthError(
-    execution: FoundRunExecution,
-    data: RunJobUnresolvedAuthError
-  ) {
-    return await $transaction(this.#prismaClient, async (tx) => {
-      await this.#failRunExecution(tx, execution, data.issues, "UNRESOLVED_AUTH");
-    });
-  }
-
-  async #failRunWithInvalidPayloadError(
-    execution: FoundRunExecution,
-    data: RunJobInvalidPayloadError
-  ) {
-    return await $transaction(this.#prismaClient, async (tx) => {
-      await this.#failRunExecution(tx, execution, data.errors, "INVALID_PAYLOAD");
-    });
-  }
-
-  async #retryRunWithTask(execution: FoundRunExecution, data: RunJobRetryWithTask) {
-    const { run } = execution;
-
-    return await $transaction(this.#prismaClient, async (tx) => {
-      // We need to check for an existing task attempt
-      const existingAttempt = await tx.taskAttempt.findFirst({
-        where: {
-          taskId: data.task.id,
-          status: "PENDING",
-        },
-        orderBy: {
-          number: "desc",
-        },
-      });
-
-      if (existingAttempt) {
-        await tx.taskAttempt.update({
-          where: {
-            id: existingAttempt.id,
-          },
-          data: {
-            status: "ERRORED",
-            error: formatError(data.error),
-          },
-        });
-      }
-
-      // We need to create a new task attempt
-      await tx.taskAttempt.create({
-        data: {
-          taskId: data.task.id,
-          number: existingAttempt ? existingAttempt.number + 1 : 1,
-          status: "PENDING",
-          runAt: data.retryAt,
-        },
-      });
-
-      await tx.task.update({
-        where: {
-          id: data.task.id,
-        },
-        data: {
-          status: "WAITING",
-        },
-      });
-
-      // Now we need to create a new job execution
-      const newJobExecution = await tx.jobRunExecution.create({
-        data: {
-          runId: run.id,
-          reason: "EXECUTE_JOB",
-          status: "PENDING",
-          retryLimit: EXECUTE_JOB_RETRY_LIMIT,
-          resumeTaskId: data.task.id,
-        },
-      });
-
-      await enqueueRunExecutionV1(
-        newJobExecution,
-        run.queue.id,
-        run.queue.maxJobs,
-        tx,
-        data.retryAt
-      );
-    });
-  }
-
-  async #failRunExecutionWithRetry(
-    execution: FoundRunExecution,
-    output: Record<string, any>
-  ): Promise<void> {
-    await $transaction(this.#prismaClient, async (tx) => {
-      if (execution.retryCount + 1 > execution.retryLimit) {
-        // We've reached the retry limit, so we need to fail the execution and stop retrying
-        return await this.#failRunExecution(tx, execution, output);
-      }
-
-      // We need to retry execution
-      const retryCount = execution.retryCount + 1;
-      // Use an exponential backoff strategy with the exponent being 1.5
-      // So when retryCount is 1, retryDelayInMs is 500ms
-      // When retryCount is 2, retryDelayInMs is 750ms
-      // When retryCount is 3, retryDelayInMs is 1125ms
-      // When retryCount is 4, retryDelayInMs is 1687ms
-      // When retryCount is 5, retryDelayInMs is 2531ms
-      // When retryCount is 6, retryDelayInMs is 3796ms
-      // When retryCount is 7, retryDelayInMs is 5694ms
-      // When retryCount is 8, retryDelayInMs is 8541ms
-      // When retryCount is 9, retryDelayInMs is 12812ms
-      // When retryCount is 10, retryDelayInMs is 19218ms
-      const retryDelayInMs = Math.round(500 * Math.pow(1.5, retryCount - 1));
-
-      await tx.jobRunExecution.update({
-        where: {
-          id: execution.id,
-        },
-        data: {
-          retryCount,
-          retryDelayInMs,
-          error: JSON.stringify(output),
-        },
-      });
-
-      const runAt = new Date(Date.now() + retryDelayInMs);
-
-      await enqueueRunExecutionV1(
-        execution,
-        execution.run.queue.id,
-        execution.run.queue.maxJobs,
-        tx,
-        runAt
-      );
-    });
-  }
-
-  async #failRunExecution(
-    prisma: PrismaClientOrTransaction,
-    execution: FoundRunExecution,
-    output: Record<string, any>,
-    status: "FAILURE" | "ABORTED" | "UNRESOLVED_AUTH" | "INVALID_PAYLOAD" = "FAILURE"
-  ): Promise<void> {
-    const { run } = execution;
-
-    await $transaction(prisma, async (tx) => {
-      switch (execution.reason) {
-        case "EXECUTE_JOB": {
-          // If the execution is an EXECUTE_JOB reason, we need to fail the run
-          await tx.jobRun.update({
-            where: { id: run.id },
-            data: {
-              completedAt: new Date(),
-              status,
-              output,
-              queue: {
-                update: {
-                  jobCount: {
-                    decrement: 1,
-                  },
-                },
-              },
-            },
-          });
-
-          break;
-        }
-        case "PREPROCESS": {
-          // If the status is ABORTED, we need to fail the run
-          if (status === "ABORTED") {
-            await tx.jobRun.update({
-              where: { id: run.id },
-              data: {
-                completedAt: new Date(),
-                status,
-                output,
-                queue: {
-                  update: {
-                    jobCount: {
-                      decrement: 1,
-                    },
-                  },
-                },
-              },
-            });
-
-            break;
-          }
-
-          await tx.jobRun.update({
-            where: {
-              id: run.id,
-            },
-            data: {
-              status: "STARTED",
-              startedAt: new Date(),
-            },
-          });
-
-          const runExecution = await tx.jobRunExecution.create({
-            data: {
-              runId: run.id,
-              reason: "EXECUTE_JOB",
-              status: "PENDING",
-              retryLimit: EXECUTE_JOB_RETRY_LIMIT,
-            },
-          });
-
-          await enqueueRunExecutionV1(runExecution, run.queue.id, run.queue.maxJobs, tx);
-
-          break;
-        }
-      }
-
-      await tx.jobRunExecution.update({
-        where: {
-          id: execution.id,
-        },
-        data: {
-          status: "FAILURE",
-          completedAt: new Date(),
-          error: JSON.stringify(output),
-        },
-      });
-    });
-  }
-
-  async #cancelExecution(execution: FoundRunExecution) {
-    await this.#prismaClient.jobRunExecution.update({
-      where: {
-        id: execution.id,
-      },
-      data: {
-        status: "FAILURE",
-        completedAt: new Date(),
-        error: "This never ran because it was canceled by the user.",
-      },
-    });
-  }
-}
-
-async function findRunExecution(prisma: PrismaClientOrTransaction, id: string) {
-  return await prisma.jobRunExecution.findUnique({
-    where: { id },
-    include: {
-      run: {
-        include: {
-          environment: true,
-          endpoint: true,
-          organization: true,
-          externalAccount: true,
-          queue: true,
-          runConnections: {
-            include: {
-              integration: true,
-              connection: {
-                include: {
-                  dataReference: true,
-                },
-              },
-            },
-          },
-          tasks: {
-            where: {
-              status: {
-                in: ["COMPLETED"],
-              },
-            },
-          },
-          event: true,
-          version: {
-            include: {
-              job: true,
-              organization: true,
-            },
-          },
-        },
-      },
-    },
-  });
-}

apps/webapp/app/services/runs/performRunExecutionV2.server.ts

@@ -3,6 +3,7 @@ import {
   BloomFilter,
   ConnectionAuth,
   EndpointHeadersSchema,
+  RunJobAutoYieldWithCompletedTaskExecutionError,
   RunJobError,
   RunJobInvalidPayloadError,
   RunJobResumeWithTask,
@@ -24,9 +25,17 @@ import { safeJsonZodParse } from "~/utils/json";
 import { EndpointApi } from "../endpointApi.server";
 import { logger } from "../logger.server";
 import { prepareTasksForCaching, prepareTasksForCachingLegacy } from "~/models/task.server";
-import { MAX_RUN_YIELDED_EXECUTIONS } from "~/consts";
+import {
+  MAX_RUN_CHUNK_EXECUTION_LIMIT,
+  MAX_RUN_YIELDED_EXECUTIONS,
+  RESPONSE_TIMEOUT_STATUS_CODES,
+  RUN_CHUNK_EXECUTION_BUFFER,
+} from "~/consts";
 import { ApiEventLog } from "@trigger.dev/core";
 import { RunJobBody } from "@trigger.dev/core";
+import { CompleteRunTaskService } from "~/routes/api.v1.runs.$runId.tasks.$id.complete";
+import { detectResponseIsTimeout } from "~/models/endpoint.server";
+import { forceYieldCoordinator } from "./forceYieldCoordinator.server";
 
 type FoundRun = NonNullable<Awaited<ReturnType<typeof findRun>>>;
 type FoundTask = FoundRun["tasks"][number];
@@ -148,6 +157,7 @@ export class PerformRunExecutionV2Service {
             status: "STARTED",
             startedAt: new Date(),
             properties: safeBody.data.properties,
+            forceYieldImmediately: false,
           },
         });
 
@@ -158,257 +168,291 @@ export class PerformRunExecutionV2Service {
     }
   }
   async #executeJob(run: FoundRun, input: PerformRunExecutionV2Input) {
-    const { isRetry, resumeTaskId } = input;
-
-    if (run.status === "CANCELED") {
-      await this.#cancelExecution(run);
-      return;
-    }
-
     try {
-      if (
-        typeof process.env.BLOCKED_ORGS === "string" &&
-        process.env.BLOCKED_ORGS.includes(run.organizationId)
-      ) {
-        logger.debug("Skipping execution for blocked org", {
-          orgId: run.organizationId,
-        });
-
-        await this.#prismaClient.jobRun.update({
-          where: {
-            id: run.id,
-          },
-          data: {
-            status: "CANCELED",
-            completedAt: new Date(),
-          },
-        });
+      const { isRetry, resumeTaskId } = input;
 
+      if (run.status === "CANCELED") {
+        await this.#cancelExecution(run);
         return;
       }
-    } catch (e) {}
 
-    const client = new EndpointApi(run.environment.apiKey, run.endpoint.url);
-    const event = eventRecordToApiJson(run.event);
+      try {
+        if (
+          typeof process.env.BLOCKED_ORGS === "string" &&
+          process.env.BLOCKED_ORGS.includes(run.organizationId)
+        ) {
+          logger.debug("Skipping execution for blocked org", {
+            orgId: run.organizationId,
+          });
 
-    const startedAt = new Date();
+          await this.#prismaClient.jobRun.update({
+            where: {
+              id: run.id,
+            },
+            data: {
+              status: "CANCELED",
+              completedAt: new Date(),
+            },
+          });
 
-    const { executionCount } = await this.#prismaClient.jobRun.update({
-      where: {
-        id: run.id,
-      },
-      data: {
-        status: run.status === "QUEUED" ? "STARTED" : run.status,
-        startedAt: run.startedAt ?? new Date(),
-        executionCount: {
-          increment: 1,
+          return;
+        }
+      } catch (e) {}
+
+      const client = new EndpointApi(run.environment.apiKey, run.endpoint.url);
+      const event = eventRecordToApiJson(run.event);
+
+      const startedAt = new Date();
+
+      const { executionCount } = await this.#prismaClient.jobRun.update({
+        where: {
+          id: run.id,
         },
-      },
-      select: {
-        executionCount: true,
-      },
-    });
-
-    const connections = await resolveRunConnections(run.runConnections);
-
-    if (!connections.success) {
-      return this.#failRunExecution(this.#prismaClient, "EXECUTE_JOB", run, {
-        message: `Could not resolve all connections for run ${run.id}. This should not happen`,
-      });
-    }
-
-    let resumedTask: Task | undefined;
-
-    if (resumeTaskId) {
-      resumedTask =
-        (await this.#prismaClient.task.findUnique({
-          where: {
-            id: resumeTaskId,
+        data: {
+          status: run.status === "QUEUED" ? "STARTED" : run.status,
+          startedAt: run.startedAt ?? new Date(),
+          executionCount: {
+            increment: 1,
           },
-        })) ?? undefined;
+        },
+        select: {
+          executionCount: true,
+        },
+      });
 
-      if (resumedTask) {
-        resumedTask = await this.#prismaClient.task.update({
+      const connections = await resolveRunConnections(run.runConnections);
+
+      if (!connections.success) {
+        return this.#failRunExecution(this.#prismaClient, "EXECUTE_JOB", run, {
+          message: `Could not resolve all connections for run ${run.id}. This should not happen`,
+        });
+      }
+
+      let resumedTask: Task | undefined;
+
+      if (resumeTaskId) {
+        resumedTask =
+          (await this.#prismaClient.task.findUnique({
+            where: {
+              id: resumeTaskId,
+            },
+          })) ?? undefined;
+
+        if (resumedTask) {
+          resumedTask = await this.#prismaClient.task.update({
+            where: {
+              id: resumeTaskId,
+            },
+            data: {
+              status: resumedTask.noop ? "COMPLETED" : "RUNNING",
+              completedAt: resumedTask.noop ? new Date() : undefined,
+            },
+          });
+        }
+      }
+
+      const sourceContext = RunSourceContextSchema.safeParse(run.event.sourceContext);
+
+      const executionBody = await this.#createExecutionBody(
+        run,
+        [run.tasks, resumedTask].flat().filter(Boolean),
+        startedAt,
+        isRetry,
+        connections.auth,
+        event,
+        sourceContext.success ? sourceContext.data : undefined
+      );
+
+      forceYieldCoordinator.registerRun(run.id);
+
+      const { response, parser, errorParser, durationInMs } = await client.executeJobRequest(
+        executionBody
+      );
+
+      forceYieldCoordinator.deregisterRun(run.id);
+
+      if (!response) {
+        return await this.#failRunExecutionWithRetry({
+          message: `Connection could not be established to the endpoint (${run.endpoint.url})`,
+        });
+      }
+
+      // Update the endpoint version if it has changed
+      const rawHeaders = Object.fromEntries(response.headers.entries());
+      const headers = EndpointHeadersSchema.safeParse(rawHeaders);
+
+      if (
+        headers.success &&
+        headers.data["trigger-version"] &&
+        headers.data["trigger-version"] !== run.endpoint.version
+      ) {
+        await this.#prismaClient.endpoint.update({
           where: {
-            id: resumeTaskId,
+            id: run.endpoint.id,
           },
           data: {
-            status: resumedTask.noop ? "COMPLETED" : "RUNNING",
-            completedAt: resumedTask.noop ? new Date() : undefined,
+            version: headers.data["trigger-version"],
           },
         });
       }
-    }
 
-    const sourceContext = RunSourceContextSchema.safeParse(run.event.sourceContext);
+      const rawBody = await response.text();
 
-    const executionBody = await this.#createExecutionBody(
-      run,
-      [run.tasks, resumedTask].flat().filter(Boolean),
-      startedAt,
-      isRetry,
-      connections.auth,
-      event,
-      sourceContext.success ? sourceContext.data : undefined
-    );
+      if (!response.ok) {
+        logger.debug("Endpoint responded with non-200 status code", {
+          status: response.status,
+          runId: run.id,
+          endpoint: run.endpoint.url,
+        });
 
-    const { response, parser, errorParser, durationInMs } = await client.executeJobRequest(
-      executionBody
-    );
+        const errorBody = safeJsonZodParse(errorParser, rawBody);
 
-    if (!response) {
-      return await this.#failRunExecutionWithRetry({
-        message: `Connection could not be established to the endpoint (${run.endpoint.url})`,
-      });
-    }
+        if (errorBody && errorBody.success) {
+          // Only retry if the error isn't a 4xx
+          if (response.status >= 400 && response.status <= 499) {
+            return await this.#failRunExecution(
+              this.#prismaClient,
+              "EXECUTE_JOB",
+              run,
+              errorBody.data
+            );
+          } else {
+            return await this.#failRunExecutionWithRetry(errorBody.data);
+          }
+        }
 
-    // Update the endpoint version if it has changed
-    const rawHeaders = Object.fromEntries(response.headers.entries());
-    const headers = EndpointHeadersSchema.safeParse(rawHeaders);
-
-    if (
-      headers.success &&
-      headers.data["trigger-version"] &&
-      headers.data["trigger-version"] !== run.endpoint.version
-    ) {
-      await this.#prismaClient.endpoint.update({
-        where: {
-          id: run.endpoint.id,
-        },
-        data: {
-          version: headers.data["trigger-version"],
-        },
-      });
-    }
-
-    const rawBody = await response.text();
-
-    if (!response.ok) {
-      logger.debug("Endpoint responded with non-200 status code", {
-        status: response.status,
-        runId: run.id,
-        endpoint: run.endpoint.url,
-      });
-
-      const errorBody = safeJsonZodParse(errorParser, rawBody);
-
-      if (errorBody && errorBody.success) {
         // Only retry if the error isn't a 4xx
-        if (response.status >= 400 && response.status <= 499) {
+        if (response.status >= 400 && response.status <= 499 && response.status !== 408) {
           return await this.#failRunExecution(
             this.#prismaClient,
             "EXECUTE_JOB",
             run,
-            errorBody.data
+            {
+              message: `Endpoint responded with ${response.status} status code`,
+            },
+            "FAILURE",
+            durationInMs
           );
         } else {
-          return await this.#failRunExecutionWithRetry(errorBody.data);
+          // If the error is a timeout, we should mark this execution as succeeded (by not throwing an error) and enqueue a new execution
+          if (detectResponseIsTimeout(response)) {
+            return await this.#resumeRunExecutionAfterTimeout(
+              this.#prismaClient,
+              run,
+              input,
+              durationInMs,
+              executionCount
+            );
+          } else {
+            return await this.#failRunExecutionWithRetry({
+              message: `Endpoint responded with ${response.status} status code`,
+            });
+          }
         }
       }
 
-      // Only retry if the error isn't a 4xx
-      if (response.status >= 400 && response.status <= 499 && response.status !== 408) {
+      const safeBody = safeJsonZodParse(parser, rawBody);
+
+      if (!safeBody) {
         return await this.#failRunExecution(
           this.#prismaClient,
           "EXECUTE_JOB",
           run,
           {
-            message: `Endpoint responded with ${response.status} status code`,
+            message: "Endpoint responded with invalid JSON",
           },
           "FAILURE",
           durationInMs
         );
-      } else {
-        // If the error is a 504 timeout, we should mark this execution as succeeded (by not throwing an error) and enqueue a new execution
-        if (response.status === 504) {
-          return await this.#resumeRunExecutionAfterTimeout(
-            this.#prismaClient,
+      }
+
+      if (!safeBody.success) {
+        return await this.#failRunExecution(
+          this.#prismaClient,
+          "EXECUTE_JOB",
+          run,
+          {
+            message: generateErrorMessage(safeBody.error.issues),
+          },
+          "FAILURE",
+          durationInMs
+        );
+      }
+
+      const status = safeBody.data.status;
+
+      switch (status) {
+        case "SUCCESS": {
+          await this.#completeRunWithSuccess(run, safeBody.data, durationInMs);
+
+          break;
+        }
+        case "RESUME_WITH_TASK": {
+          await this.#resumeRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
+
+          break;
+        }
+        case "ERROR": {
+          await this.#failRunWithError(run, safeBody.data, durationInMs);
+
+          break;
+        }
+        case "RETRY_WITH_TASK": {
+          await this.#retryRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
+
+          break;
+        }
+        case "CANCELED": {
+          await this.#cancelExecution(run);
+          break;
+        }
+        case "UNRESOLVED_AUTH_ERROR": {
+          await this.#failRunWithUnresolvedAuthError(run, safeBody.data, durationInMs);
+
+          break;
+        }
+        case "INVALID_PAYLOAD": {
+          await this.#failRunWithInvalidPayloadError(run, safeBody.data, durationInMs);
+
+          break;
+        }
+        case "YIELD_EXECUTION": {
+          await this.#resumeYieldedRun(
             run,
-            input,
+            safeBody.data.key,
+            isRetry,
             durationInMs,
             executionCount
           );
-        } else {
-          return await this.#failRunExecutionWithRetry({
-            message: `Endpoint responded with ${response.status} status code`,
-          });
+          break;
+        }
+        case "AUTO_YIELD_EXECUTION": {
+          await this.#resumeAutoYieldedRun(
+            run,
+            safeBody.data,
+            isRetry,
+            durationInMs,
+            executionCount
+          );
+          break;
+        }
+        case "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK": {
+          await this.#resumeAutoYieldedRunWithCompletedTask(
+            run,
+            safeBody.data,
+            isRetry,
+            durationInMs,
+            executionCount
+          );
+          break;
+        }
+        default: {
+          const _exhaustiveCheck: never = status;
+          throw new Error(`Non-exhaustive match for value: ${status}`);
         }
       }
-    }
-
-    const safeBody = safeJsonZodParse(parser, rawBody);
-
-    if (!safeBody) {
-      return await this.#failRunExecution(
-        this.#prismaClient,
-        "EXECUTE_JOB",
-        run,
-        {
-          message: "Endpoint responded with invalid JSON",
-        },
-        "FAILURE",
-        durationInMs
-      );
-    }
-
-    if (!safeBody.success) {
-      return await this.#failRunExecution(
-        this.#prismaClient,
-        "EXECUTE_JOB",
-        run,
-        {
-          message: generateErrorMessage(safeBody.error.issues),
-        },
-        "FAILURE",
-        durationInMs
-      );
-    }
-
-    const status = safeBody.data.status;
-
-    switch (status) {
-      case "SUCCESS": {
-        await this.#completeRunWithSuccess(run, safeBody.data, durationInMs);
-
-        break;
-      }
-      case "RESUME_WITH_TASK": {
-        await this.#resumeRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
-
-        break;
-      }
-      case "ERROR": {
-        await this.#failRunWithError(run, safeBody.data, durationInMs);
-
-        break;
-      }
-      case "RETRY_WITH_TASK": {
-        await this.#retryRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
-
-        break;
-      }
-      case "CANCELED": {
-        await this.#cancelExecution(run);
-        break;
-      }
-      case "UNRESOLVED_AUTH_ERROR": {
-        await this.#failRunWithUnresolvedAuthError(run, safeBody.data, durationInMs);
-
-        break;
-      }
-      case "INVALID_PAYLOAD": {
-        await this.#failRunWithInvalidPayloadError(run, safeBody.data, durationInMs);
-
-        break;
-      }
-      case "YIELD_EXECUTION": {
-        await this.#resumeYieldedRun(run, safeBody.data.key, isRetry, durationInMs, executionCount);
-        break;
-      }
-      default: {
-        const _exhaustiveCheck: never = status;
-        throw new Error(`Non-exhaustive match for value: ${status}`);
-      }
+    } finally {
+      forceYieldCoordinator.deregisterRun(run.id);
     }
   }
 
@@ -458,6 +502,13 @@ export class PerformRunExecutionV2Service {
         cachedTaskCursor: preparedTasks.cursor,
         noopTasksSet: prepareNoOpTasksBloomFilter(tasks),
         yieldedExecutions: run.yieldedExecutions,
+        runChunkExecutionLimit: run.endpoint.runChunkExecutionLimit - RUN_CHUNK_EXECUTION_BUFFER,
+        autoYieldConfig: {
+          startTaskThreshold: run.endpoint.startTaskThreshold,
+          beforeExecuteTaskThreshold: run.endpoint.beforeExecuteTaskThreshold,
+          beforeCompleteTaskThreshold: run.endpoint.beforeCompleteTaskThreshold,
+          afterCompleteTaskThreshold: run.endpoint.afterCompleteTaskThreshold,
+        },
       };
     }
 
@@ -639,6 +690,7 @@ export class PerformRunExecutionV2Service {
           yieldedExecutions: {
             push: key,
           },
+          forceYieldImmediately: false,
         },
         select: {
           yieldedExecutions: true,
@@ -654,6 +706,101 @@ export class PerformRunExecutionV2Service {
     });
   }
 
+  async #resumeAutoYieldedRun(
+    run: FoundRun,
+    data: { location: string; timeRemaining: number; timeElapsed: number; limit?: number },
+    isRetry: boolean,
+    durationInMs: number,
+    executionCount: number
+  ) {
+    await $transaction(this.#prismaClient, async (tx) => {
+      await tx.jobRun.update({
+        where: {
+          id: run.id,
+        },
+        data: {
+          executionDuration: {
+            increment: durationInMs,
+          },
+          executionCount: {
+            increment: 1,
+          },
+          autoYieldExecution: {
+            create: [
+              {
+                location: data.location,
+                timeRemaining: data.timeRemaining,
+                timeElapsed: data.timeElapsed,
+                limit: data.limit ?? 0,
+              },
+            ],
+          },
+          forceYieldImmediately: false,
+        },
+        select: {
+          executionCount: true,
+        },
+      });
+
+      await enqueueRunExecutionV2(run, tx, {
+        isRetry,
+        skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
+        executionCount,
+      });
+    });
+  }
+
+  async #resumeAutoYieldedRunWithCompletedTask(
+    run: FoundRun,
+    data: RunJobAutoYieldWithCompletedTaskExecutionError,
+    isRetry: boolean,
+    durationInMs: number,
+    executionCount: number
+  ) {
+    await $transaction(this.#prismaClient, async (tx) => {
+      await tx.jobRun.update({
+        where: {
+          id: run.id,
+        },
+        data: {
+          executionDuration: {
+            increment: durationInMs,
+          },
+          executionCount: {
+            increment: 1,
+          },
+          autoYieldExecution: {
+            create: [
+              {
+                location: data.data.location,
+                timeRemaining: data.data.timeRemaining,
+                timeElapsed: data.data.timeElapsed,
+                limit: data.data.limit ?? 0,
+              },
+            ],
+          },
+          forceYieldImmediately: false,
+        },
+        select: {
+          executionCount: true,
+        },
+      });
+
+      const service = new CompleteRunTaskService(tx);
+
+      await service.call(run.environment, run.id, data.id, {
+        properties: data.properties,
+        output: data.output,
+      });
+
+      await enqueueRunExecutionV2(run, tx, {
+        isRetry,
+        skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
+        executionCount,
+      });
+    });
+  }
+
   async #retryRunWithTask(
     run: FoundRun,
     data: RunJobRetryWithTask,
@@ -748,6 +895,54 @@ export class PerformRunExecutionV2Service {
         return;
       }
 
+      const runWithLatestTask = await tx.jobRun.findUniqueOrThrow({
+        where: {
+          id: run.id,
+        },
+        select: {
+          tasks: {
+            select: {
+              id: true,
+              name: true,
+              status: true,
+              displayKey: true,
+            },
+            take: 1,
+            orderBy: { createdAt: "desc" },
+          },
+          _count: {
+            select: {
+              tasks: true,
+            },
+          },
+        },
+      });
+
+      if (runWithLatestTask._count.tasks === run._count.tasks) {
+        const latestTask = runWithLatestTask.tasks[0];
+
+        const cause =
+          latestTask?.status === "RUNNING"
+            ? `This is likely caused by task "${
+                latestTask.displayKey ?? latestTask.name
+              }" execution exceeding the function timeout`
+            : "This is likely caused by executing code outside of a task that exceeded the function timeout";
+
+        await this.#failRunExecution(
+          tx,
+          "EXECUTE_JOB",
+          run,
+          {
+            message: `Function timeout detected in ${
+              durationInMs / 1000.0
+            }s without any task creation. This is unexpected behavior and could lead to an infinite execution error because the run will never finish. ${cause}`,
+          },
+          "TIMED_OUT",
+          durationInMs
+        );
+        return;
+      }
+
       await tx.jobRun.update({
         where: {
           id: run.id,
@@ -756,6 +951,16 @@ export class PerformRunExecutionV2Service {
           executionDuration: {
             increment: durationInMs,
           },
+          endpoint: {
+            update: {
+              // Never allow the execution limit to be less than 10 seconds or more than MAX_RUN_CHUNK_EXECUTION_LIMIT
+              runChunkExecutionLimit: Math.min(
+                Math.max(durationInMs, 10000),
+                MAX_RUN_CHUNK_EXECUTION_LIMIT
+              ),
+            },
+          },
+          forceYieldImmediately: false,
         },
       });
 
@@ -794,6 +999,20 @@ export class PerformRunExecutionV2Service {
               executionDuration: {
                 increment: durationInMs,
               },
+              tasks: {
+                updateMany: {
+                  where: {
+                    status: {
+                      in: ["WAITING", "RUNNING", "PENDING"],
+                    },
+                  },
+                  data: {
+                    status: status === "TIMED_OUT" ? "CANCELED" : "ERRORED",
+                    completedAt: new Date(),
+                  },
+                },
+              },
+              forceYieldImmediately: false,
             },
           });
 
@@ -855,7 +1074,12 @@ async function findRun(prisma: PrismaClientOrTransaction, id: string) {
   return await prisma.jobRun.findUnique({
     where: { id },
     include: {
-      environment: true,
+      environment: {
+        include: {
+          project: true,
+          organization: true,
+        },
+      },
       endpoint: true,
       organization: true,
       externalAccount: true,
@@ -894,6 +1118,11 @@ async function findRun(prisma: PrismaClientOrTransaction, id: string) {
           organization: true,
         },
       },
+      _count: {
+        select: {
+          tasks: true,
+        },
+      },
     },
   });
 }

apps/webapp/app/services/schedules/nextScheduledEvent.server.ts

@@ -36,7 +36,7 @@ export class NextScheduledEventService {
 
       const scheduleTime = calculateNextScheduledEvent(
         schedule.data,
-        scheduleSource.lastEventTimestamp
+        scheduleSource.lastEventTimestamp ?? scheduleSource.createdAt
       );
 
       logger.debug("enqueuing scheduled event", {
@@ -67,6 +67,7 @@ export class NextScheduledEventService {
         },
         data: {
           workerJobId: workerJob.id,
+          nextEventTimestamp: scheduleTime,
         },
       });
 

apps/webapp/app/services/sources/registerSourceV2.server.ts

@@ -6,6 +6,7 @@ import type { AuthenticatedEnvironment } from "../apiAuth.server";
 import { workerQueue } from "../worker.server";
 import { generateSecret } from "./utils.server";
 import { ExtendedEndpoint, findEndpoint } from "~/models/endpoint.server";
+import { logger } from "../logger.server";
 
 export class RegisterSourceServiceV2 {
   #prismaClient: PrismaClientOrTransaction;
@@ -211,16 +212,22 @@ export class RegisterSourceServiceV2 {
 
         // Collect the options that are no longer being used so we can remove them
         const newOptions = metadata.options;
-        const orphanedOptions: Record<string, string[]> = {};
-        for (const event of triggerSource.options) {
-          const values = newOptions[event.name];
-          if (values === undefined) {
-            orphanedOptions[event.name] = [event.value];
+        const orphanedOptions: Record<string, Set<string>> = {};
+        for (const option of triggerSource.options) {
+          const newValues = newOptions[option.name];
+
+          // initialize the set
+          if (!orphanedOptions[option.name]) {
+            orphanedOptions[option.name] = new Set();
+          }
+
+          if (newValues === undefined) {
+            orphanedOptions[option.name] = new Set([...orphanedOptions[option.name], option.value]);
             continue;
           }
 
-          if (values!.includes(event.value)) {
-            orphanedOptions[event.name] = [...values, event.value];
+          if (!newValues.includes(option.value)) {
+            orphanedOptions[option.name] = new Set([...orphanedOptions[option.name], option.value]);
           }
         }
 
@@ -250,9 +257,26 @@ export class RegisterSourceServiceV2 {
           });
         }
 
+        // Delete the orphaned options
+        for (const [name, values] of Object.entries(orphanedOptions)) {
+          for (const value of values) {
+            await tx.triggerSourceOption.delete({
+              where: {
+                name_value_sourceId: {
+                  name,
+                  value,
+                  sourceId: triggerSource.id,
+                },
+              },
+            });
+          }
+        }
+
         return {
           id: triggerSource.id,
-          orphanedOptions,
+          orphanedOptions: Object.fromEntries(
+            Object.entries(orphanedOptions).map(([name, values]) => [name, Array.from(values)])
+          ),
         };
       },
       { timeout: 15000 }
@@ -284,9 +308,19 @@ export class RegisterSourceServiceV2 {
     }
 
     const triggerIsActive = triggerSource.active;
-    const triggerHasOrphanedEvents = Object.keys(orphanedOptions).length > 0;
+    const triggerHasOrphanedEvents = Object.values(orphanedOptions).some(
+      (values) => values.length > 0
+    );
     const triggerHasUnregisteredEvents = triggerSource.options.some((option) => !option.registered);
 
+    logger.debug("Deciding whether to activate source", {
+      triggerIsActive,
+      triggerHasOrphanedEvents,
+      triggerHasUnregisteredEvents,
+      orphanedOptions,
+      options: triggerSource.options,
+    });
+
     if (!triggerIsActive || triggerHasOrphanedEvents || triggerHasUnregisteredEvents) {
       // We need to re-activate the source, and there could be orphaned events
       await workerQueue.enqueue("activateSource", {

apps/webapp/app/services/tasks/performTaskOperation.server.ts

@@ -35,7 +35,7 @@ export class PerformTaskOperationService {
     }
 
     if (!task.operation) {
-      return await this.#resumeTask(task, null);
+      return await this.#resumeTask(task, null, 0);
     }
 
     logger.debug("PerformTaskOperationService.call", { task });
@@ -53,12 +53,16 @@ export class PerformTaskOperationService {
 
         const { url, requestInit, retry } = fetchOperation.data;
 
+        const startTimeInMs = performance.now();
+
         const response = await fetch(url, {
           method: requestInit?.method ?? "GET",
           headers: normalizeHeaders(requestInit?.headers ?? {}),
           body: requestInit?.body,
         });
 
+        const durationInMs = Math.floor(performance.now() - startTimeInMs);
+
         const jsonBody = await safeJsonFromResponse(response);
 
         logger.debug("PerformTaskOperationService.call.fetch", {
@@ -68,6 +72,7 @@ export class PerformTaskOperationService {
           statusCode: response.status,
           headers: Object.fromEntries(response.headers.entries()),
           jsonBody,
+          durationInMs,
         });
 
         if (!response.ok) {
@@ -91,7 +96,7 @@ export class PerformTaskOperationService {
           }
         }
 
-        return await this.#resumeTask(task, jsonBody);
+        return await this.#resumeTask(task, jsonBody, durationInMs);
       }
       default: {
         await this.#resumeTaskWithError(task, {
@@ -118,6 +123,8 @@ export class PerformTaskOperationService {
 
     logger.debug("Calculating retry at for strategy", {
       strategy,
+      status: response.status,
+      retry,
     });
 
     switch (strategy.strategy) {
@@ -218,7 +225,7 @@ export class PerformTaskOperationService {
     });
   }
 
-  async #resumeTask(task: NonNullable<FoundTask>, output: any) {
+  async #resumeTask(task: NonNullable<FoundTask>, output: any, durationInMs: number) {
     await $transaction(this.#prismaClient, async (tx) => {
       await tx.taskAttempt.updateMany({
         where: {
@@ -236,6 +243,13 @@ export class PerformTaskOperationService {
           status: "COMPLETED",
           completedAt: new Date(),
           output: output ? output : undefined,
+          run: {
+            update: {
+              executionDuration: {
+                increment: durationInMs,
+              },
+            },
+          },
         },
       });
 

apps/webapp/app/services/worker.server.ts

@@ -1,18 +1,18 @@
 import { DeliverEmailSchema } from "@/../../packages/emails/src";
-import { ScheduledPayloadSchema } from "@trigger.dev/core";
+import { ScheduledPayloadSchema, addMissingVersionField } from "@trigger.dev/core";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
 import { ZodWorker } from "~/platform/zodWorker.server";
 import { sendEmail } from "./email.server";
 import { IndexEndpointService } from "./endpoints/indexEndpoint.server";
+import { PerformEndpointIndexService } from "./endpoints/performEndpointIndexService";
 import { RecurringEndpointIndexService } from "./endpoints/recurringEndpointIndex.server";
 import { DeliverEventService } from "./events/deliverEvent.server";
 import { InvokeDispatcherService } from "./events/invokeDispatcher.server";
 import { integrationAuthRepository } from "./externalApis/integrationAuthRepository.server";
 import { IntegrationConnectionCreatedService } from "./externalApis/integrationConnectionCreated.server";
 import { MissingConnectionCreatedService } from "./runs/missingConnectionCreated.server";
-import { PerformRunExecutionV1Service } from "./runs/performRunExecutionV1.server";
 import { PerformRunExecutionV2Service } from "./runs/performRunExecutionV2.server";
 import { StartRunService } from "./runs/startRun.server";
 import { DeliverScheduledEventService } from "./schedules/deliverScheduledEvent.server";
@@ -20,7 +20,7 @@ import { ActivateSourceService } from "./sources/activateSource.server";
 import { DeliverHttpSourceRequestService } from "./sources/deliverHttpSourceRequest.server";
 import { PerformTaskOperationService } from "./tasks/performTaskOperation.server";
 import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout";
-import { addMissingVersionField } from "@trigger.dev/core";
+import { ProbeEndpointService } from "./endpoints/probeEndpoint.server";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -29,6 +29,9 @@ const workerCatalog = {
     sourceData: z.any().optional(),
     reason: z.string().optional(),
   }),
+  performEndpointIndexing: z.object({
+    id: z.string(),
+  }),
   scheduleEmail: DeliverEmailSchema,
   startRun: z.object({ id: z.string() }),
   processCallbackTimeout: z.object({
@@ -72,12 +75,15 @@ const workerCatalog = {
   connectionCreated: z.object({
     id: z.string(),
   }),
+  probeEndpoint: z.object({
+    id: z.string(),
+  }),
+  simulate: z.object({
+    seconds: z.number(),
+  }),
 };
 
 const executionWorkerCatalog = {
-  performRunExecution: z.object({
-    id: z.string(),
-  }),
   performRunExecutionV2: z.object({
     id: z.string(),
     reason: z.enum(["EXECUTE_JOB", "PREPROCESS"]),
@@ -128,6 +134,11 @@ function getWorkerQueue() {
   return new ZodWorker({
     name: "workerQueue",
     prisma,
+    cleanup: {
+      frequencyExpression: "13,27,43 * * * *",
+      ttl: 7 * 24 * 60 * 60 * 1000, // 7 days
+      maxCount: 1000,
+    },
     runnerOptions: {
       connectionString: env.DATABASE_URL,
       concurrency: env.WORKER_CONCURRENCY,
@@ -136,6 +147,7 @@ function getWorkerQueue() {
       schema: env.WORKER_SCHEMA,
       maxPoolSize: env.WORKER_CONCURRENCY,
     },
+    shutdownTimeoutInMs: env.GRACEFUL_SHUTDOWN_TIMEOUT,
     schema: workerCatalog,
     recurringTasks: {
       // Run this every 5 minutes
@@ -229,6 +241,7 @@ function getWorkerQueue() {
       deliverHttpSourceRequest: {
         priority: 1, // smaller number = higher priority
         maxAttempts: 14,
+        queueName: (payload) => `sources:${payload.id}`,
         handler: async (payload, job) => {
           const service = new DeliverHttpSourceRequestService();
 
@@ -255,7 +268,6 @@ function getWorkerQueue() {
       },
       performTaskOperation: {
         priority: 0, // smaller number = higher priority
-        queueName: (payload) => `tasks:${payload.id}`,
         maxAttempts: 3,
         handler: async (payload, job) => {
           const service = new PerformTaskOperationService();
@@ -264,7 +276,6 @@ function getWorkerQueue() {
         },
       },
       scheduleEmail: {
-        queueName: "internal-queue",
         priority: 100,
         maxAttempts: 3,
         handler: async (payload, job) => {
@@ -276,10 +287,17 @@ function getWorkerQueue() {
         maxAttempts: 7,
         handler: async (payload, job) => {
           const service = new IndexEndpointService();
-
           await service.call(payload.id, payload.source, payload.reason, payload.sourceData);
         },
       },
+      performEndpointIndexing: {
+        priority: 1, // smaller number = higher priority
+        maxAttempts: 7,
+        handler: async (payload, job) => {
+          const service = new PerformEndpointIndexService();
+          await service.call(payload.id);
+        },
+      },
       deliverEvent: {
         priority: 0, // smaller number = higher priority
         maxAttempts: 5,
@@ -291,7 +309,6 @@ function getWorkerQueue() {
       },
       refreshOAuthToken: {
         priority: 8, // smaller number = higher priority
-        queueName: "internal-queue",
         maxAttempts: 7,
         handler: async (payload, job) => {
           await integrationAuthRepository.refreshConnection({
@@ -299,6 +316,21 @@ function getWorkerQueue() {
           });
         },
       },
+      probeEndpoint: {
+        priority: 10,
+        maxAttempts: 1,
+        handler: async (payload, job) => {
+          const service = new ProbeEndpointService();
+
+          await service.call(payload.id);
+        },
+      },
+      simulate: {
+        maxAttempts: 5,
+        handler: async (payload, job) => {
+          await new Promise((resolve) => setTimeout(resolve, payload.seconds * 1000));
+        },
+      },
     },
   });
 }
@@ -315,19 +347,9 @@ function getExecutionWorkerQueue() {
       schema: env.WORKER_SCHEMA,
       maxPoolSize: env.EXECUTION_WORKER_CONCURRENCY,
     },
+    shutdownTimeoutInMs: env.GRACEFUL_SHUTDOWN_TIMEOUT,
     schema: executionWorkerCatalog,
     tasks: {
-      performRunExecution: {
-        priority: 0, // smaller number = higher priority
-        maxAttempts: 1,
-        handler: async (payload, job) => {
-          // This is a legacy task that we don't use anymore, but needs to be here for backwards compatibility
-          // TODO: remove this once all performRunExecution tasks have been processed
-          const service = new PerformRunExecutionV1Service();
-
-          await service.call(payload.id);
-        },
-      },
       performRunExecutionV2: {
         priority: 0, // smaller number = higher priority
         maxAttempts: 12,

apps/webapp/app/utils/icon.ts

@@ -0,0 +1,10 @@
+import { hasIcon } from "@trigger.dev/companyicons";
+import { iconNames as namedIcons } from "~/components/primitives/NamedIcon";
+import { tablerIcons } from "~/utils/tablerIcons";
+
+export const isValidIcon = (icon?: string): boolean => {
+  if (!icon) {
+    return false;
+  }
+  return namedIcons.includes(icon) || hasIcon(icon) || tablerIcons.has(icon);
+};

apps/webapp/app/utils/sse.ts

@@ -47,7 +47,7 @@ export function sse({ request, pingInterval = 1000, updateInterval = 348, run }:
             });
           }
         } else {
-          logger.debug("Uknown error sending SSE, aborting", {
+          logger.debug("Unknown error sending SSE, aborting", {
             error,
             args,
           });

apps/webapp/package.json

@@ -13,7 +13,7 @@
     "lint": "eslint --cache --cache-location ./node_modules/.cache/eslint .",
     "start": "cross-env NODE_ENV=production node --max-old-space-size=8192 ./build/server.js",
     "start:local": "cross-env node --max-old-space-size=8192 ./build/server.js",
-    "typecheck": "tsc --noEmit",
+    "typecheck": "tsc -p ./tsconfig.check.json",
     "db:seed": "node prisma/seed.js",
     "db:seed:local": "ts-node prisma/seed.ts",
     "generate:sourcemaps": "remix build --sourcemap",
@@ -40,7 +40,6 @@
     "@codemirror/view": "^6.5.0",
     "@conform-to/react": "^0.6.1",
     "@conform-to/zod": "^0.6.1",
-    "@godaddy/terminus": "^4.12.1",
     "@headlessui/react": "^1.7.8",
     "@heroicons/react": "^2.0.12",
     "@highlight-run/node": "^3.1.0",
@@ -55,11 +54,11 @@
     "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-tabs": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.5",
-    "@remix-run/express": "1.19.2-pre.0",
-    "@remix-run/node": "1.19.2-pre.0",
-    "@remix-run/react": "1.19.2-pre.0",
-    "@remix-run/serve": "1.19.2-pre.0",
-    "@remix-run/server-runtime": "1.19.2-pre.0",
+    "@remix-run/express": "1.19.2",
+    "@remix-run/node": "1.19.2",
+    "@remix-run/react": "1.19.2",
+    "@remix-run/serve": "1.19.2",
+    "@remix-run/server-runtime": "1.19.2",
     "@team-plain/typescript-sdk": "^2.2.0",
     "@trigger.dev/companyicons": "^1.5.14",
     "@trigger.dev/core": "workspace:*",
@@ -87,7 +86,7 @@
     "morgan": "^1.10.0",
     "nanoid": "^3.3.4",
     "postcss-import": "^14.1.0",
-    "posthog-js": "^1.69.0",
+    "posthog-js": "^1.83.0",
     "posthog-node": "^3.1.1",
     "prism-react-renderer": "^1.3.5",
     "prismjs": "^1.29.0",
@@ -106,18 +105,20 @@
     "simple-oauth2": "^5.0.0",
     "simplur": "^3.0.1",
     "slug": "^6.0.0",
+    "sonner": "^1.0.3",
     "tailwind-merge": "^1.12.0",
     "tailwind-scrollbar-hide": "^1.1.7",
     "tailwindcss-animate": "^1.0.5",
     "tiny-invariant": "^1.2.0",
     "ulid": "^2.3.0",
     "zod": "3.22.3",
-    "zod-error": "1.5.0"
+    "zod-error": "1.5.0",
+    "zod-validation-error": "^1.5.0"
   },
   "devDependencies": {
-    "@remix-run/dev": "1.19.2-pre.0",
-    "@remix-run/eslint-config": "1.19.2-pre.0",
-    "@remix-run/testing": "^1.19.2-pre.0",
+    "@remix-run/dev": "1.19.2",
+    "@remix-run/eslint-config": "1.19.2",
+    "@remix-run/testing": "^1.19.2",
     "@storybook/addon-backgrounds": "^7.0.7",
     "@storybook/addon-docs": "^7.0.12",
     "@storybook/addon-essentials": "^7.0.7",
@@ -178,4 +179,4 @@
   "engines": {
     "node": ">=16.0.0"
   }
-}
+}

apps/webapp/server.ts

@@ -3,7 +3,6 @@ import express from "express";
 import compression from "compression";
 import morgan from "morgan";
 import { createRequestHandler } from "@remix-run/express";
-import { createTerminus } from "@godaddy/terminus";
 
 const app = express();
 
@@ -61,9 +60,19 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") {
     console.log(`✅ app ready: http://localhost:${port}`);
   });
 
-  // Handle shutdowns gracefully
-  createTerminus(server, { signals: ["SIGINT", "SIGTERM"], timeout: 5000 });
+  server.keepAliveTimeout = 65 * 1000;
+
+  process.on("SIGTERM", () => {
+    server.close((err) => {
+      if (err) {
+        console.error("Error closing express server:", err);
+      } else {
+        console.log("Express server closed gracefully.");
+      }
+    });
+  });
 } else {
+  require(BUILD_DIR);
   console.log(`✅ app ready (skipping http server)`);
 }
 

apps/webapp/tsconfig.check.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "paths": {
+      "~/*": ["./app/*"],
+      "@/*": ["./*"]
+    }
+  }
+}

config-packages/tsconfig/integration.json

@@ -3,6 +3,8 @@
   "compilerOptions": {
     "lib": ["DOM", "DOM.Iterable", "ES2019"],
     "paths": {
+      "@trigger.dev/tsup/*": ["../../config-packages/tsup/src/*"],
+      "@trigger.dev/tsup": ["../../config-packages/tsup/src/index"],
       "@trigger.dev/sdk/*": ["../../packages/trigger-sdk/src/*"],
       "@trigger.dev/sdk": ["../../packages/trigger-sdk/src/index"],
       "@trigger.dev/integration-kit/*": ["../../packages/integration-kit/src/*"],

config-packages/tsup/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@trigger.dev/tsup",
+  "version": "0.0.0",
+  "private": true,
+  "license": "MIT",
+  "devDependencies": {
+    "tsup": "7.1.x"
+  }
+}

config-packages/tsup/src/index.ts

@@ -0,0 +1,3 @@
+export { defineConfig } from "tsup";
+export { deepMergeOptions } from "./utils";
+export { options as integrationOptions } from "./integration";

config-packages/tsup/src/integration.ts

@@ -0,0 +1,22 @@
+import { Options, defineConfig } from "tsup";
+
+export const options: Options = {
+  name: "main",
+  entry: ["./src/index.ts"],
+  outDir: "./dist",
+  platform: "node",
+  format: ["cjs"],
+  legacyOutput: true,
+  sourcemap: true,
+  clean: true,
+  bundle: true,
+  splitting: false,
+  dts: true,
+  treeshake: {
+    preset: "smallest",
+  },
+  esbuildPlugins: [],
+  external: ["http", "https", "util", "events", "tty", "os", "timers"],
+};
+
+export default defineConfig(options);

config-packages/tsup/src/utils.ts

@@ -0,0 +1,32 @@
+import { Options } from "tsup";
+
+export const deepMergeOptions = deepMergeRecords<Options>;
+
+function deepMergeRecords<TRecord extends Record<any, any>>(...options: TRecord[]): TRecord {
+  const result = {} as TRecord;
+
+  for (const option of options) {
+    for (const key in option) {
+      if (option.hasOwnProperty(key)) {
+        const optionValue = option[key];
+        const existingValue = result[key];
+
+        if (
+          existingValue &&
+          typeof existingValue === "object" &&
+          typeof optionValue === "object" &&
+          !Array.isArray(existingValue) &&
+          !Array.isArray(optionValue) &&
+          existingValue !== null &&
+          optionValue !== null
+        ) {
+          result[key] = deepMergeRecords(existingValue, optionValue);
+        } else {
+          result[key] = optionValue;
+        }
+      }
+    }
+  }
+
+  return result;
+}

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.16.1-bullseye-slim AS pruner
+FROM node:18.18.2-bullseye-slim@sha256:a50436b44fe3eb2ff72696fb394ad3cae45f06cb2a7468c5ef3f012d23d9888f AS pruner
 
 WORKDIR /triggerdotdev
 
@@ -7,7 +7,7 @@ RUN npx -q turbo@1.10.9 prune --scope=webapp --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
 # Base strategy to have layer caching
-FROM node:18.16.1-bullseye-slim AS base
+FROM node:18.18.2-bullseye-slim@sha256:a50436b44fe3eb2ff72696fb394ad3cae45f06cb2a7468c5ef3f012d23d9888f AS base
 RUN apt-get update && apt-get install -y openssl dumb-init
 WORKDIR /triggerdotdev
 COPY --chown=node:node .gitignore .gitignore
@@ -32,7 +32,8 @@ ENV NODE_ENV production
 RUN pnpm install --prod --no-frozen-lockfile
 COPY --from=pruner --chown=node:node /triggerdotdev/packages/database/prisma/schema.prisma /triggerdotdev/packages/database/prisma/schema.prisma
 # RUN pnpm add @prisma/client@5.1.1 -w
-RUN pnpx prisma@4.16.0 generate --schema /triggerdotdev/packages/database/prisma/schema.prisma
+ENV NPM_CONFIG_IGNORE_WORKSPACE_ROOT_CHECK true
+RUN pnpx prisma@5.4.1 generate --schema /triggerdotdev/packages/database/prisma/schema.prisma
 
 ## Builder (builds the webapp)
 FROM base AS builder
@@ -49,7 +50,7 @@ RUN pnpm run generate
 RUN pnpm run build --filter=webapp...
 
 # Runner
-FROM node:18.16.1-bullseye-slim AS runner
+FROM node:18.18.2-bullseye-slim@sha256:a50436b44fe3eb2ff72696fb394ad3cae45f06cb2a7468c5ef3f012d23d9888f AS runner
 RUN apt-get update && apt-get install -y openssl
 WORKDIR /triggerdotdev
 RUN corepack enable

docker/dev-compose.yml

@@ -9,7 +9,7 @@ networks:
 
 services:
   db:
-    container_name: db
+    container_name: devdb
     image: postgres:14
     restart: always
     volumes:
@@ -30,11 +30,14 @@ services:
       - 3030:3030
     depends_on:
       - db
+    env_file:
+      - ../.env
     environment:
       DATABASE_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
       DIRECT_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
       SESSION_SECRET: secret123
       MAGIC_LINK_SECRET: secret123
+      ENCRYPTION_KEY: secret123
       REMIX_APP_PORT: 3030
       PORT: 3030
     networks:

docker/docker-compose.yml

@@ -2,6 +2,7 @@ version: "3"
 
 volumes:
   database-data:
+  pgadmin-data:
 
 networks:
   app_network:
@@ -22,3 +23,21 @@ services:
       - app_network
     ports:
       - 5432:5432
+
+  pgadmin:
+    container_name: pgadmin
+    image: dpage/pgadmin4:7
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@example.com
+      PGADMIN_DEFAULT_PASSWORD: admin
+      PGADMIN_DISABLE_POSTFIX: "true"
+    volumes:
+      - pgadmin-data:/var/lib/pgadmin
+      - ./pgadmin/servers.json:/pgadmin4/servers.json
+    networks:
+      - app_network
+    ports:
+      - 5480:80
+    depends_on:
+      - database

docker/pgadmin/servers.json

@@ -0,0 +1,13 @@
+{
+  "Servers": {
+    "1": {
+      "Name": "Trigger.dev",
+      "Group": "Trigger.dev",
+      "Port": 5432,
+      "Username": "postgres",
+      "Host": "database",
+      "SSLMode": "prefer",
+      "MaintenanceDB": "postgres"
+    }
+  }
+}

docker/scripts/entrypoint.sh

@@ -14,4 +14,6 @@ cp node_modules/@prisma/engines/*.node apps/webapp/prisma/
 pnpm --filter webapp db:seed
 
 cd /triggerdotdev/apps/webapp
-exec dumb-init pnpm run start:local
+# exec dumb-init pnpm run start:local
+NODE_PATH='/triggerdotdev/node_modules/.pnpm/node_modules' exec dumb-init node --max-old-space-size=8192 ./build/server.js
+

docker/services-compose.yml

@@ -0,0 +1,65 @@
+version: "3"
+
+volumes:
+  database-data:
+
+networks:
+  app_network:
+    external: false
+
+services:
+  db:
+    container_name: devdb
+    image: postgres:14
+    restart: always
+    volumes:
+      - database-data:/var/lib/postgresql/data/
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+    networks:
+      - app_network
+    ports:
+      - 5432:5432
+  app:
+    build:
+      context: ../
+      dockerfile: ./docker/Dockerfile
+    ports:
+      - 3030:3030
+    depends_on:
+      - db
+    env_file:
+      - ../.env
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
+      DIRECT_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
+      SESSION_SECRET: secret123
+      MAGIC_LINK_SECRET: secret123
+      ENCRYPTION_KEY: secret123
+      REMIX_APP_PORT: 3030
+      PORT: 3030
+      WORKER_ENABLED: "false"
+      EXECUTION_WORKER_ENABLED: "false"
+    networks:
+      - app_network
+  worker:
+    build:
+      context: ../
+      dockerfile: ./docker/Dockerfile
+    depends_on:
+      - db
+    env_file:
+      - ../.env
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
+      DIRECT_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
+      SESSION_SECRET: secret123
+      MAGIC_LINK_SECRET: secret123
+      ENCRYPTION_KEY: secret123
+      REMIX_APP_PORT: 3030
+      PORT: 3030
+      HTTP_SERVER_DISABLED: "true"
+    networks:
+      - app_network

docs/_snippets/manual-setup-sveltekit.mdx

@@ -1 +1,217 @@
-We're in the process of building support for the SvelteKit framework. You can follow along with progress or contribute via [this GitHub issue](https://github.com/triggerdotdev/trigger.dev/issues).
+## Installing Required Packages
+
+To begin, install the necessary packages in your Sveltekit project directory. You can choose one of the following package managers:
+
+<CodeGroup>
+
+```bash npm
+npm i @trigger.dev/sdk @trigger.dev/sveltekit
+```
+
+```bash pnpm
+pnpm install @trigger.dev/sdk @trigger.dev/sveltekit
+```
+
+```bash yarn
+yarn add @trigger.dev/sdk @trigger.dev/sveltekit
+```
+
+</CodeGroup>
+<br />
+
+<Note>Ensure that you execute this command within a SvelteKit project.</Note>
+## Obtaining the Development API Key
+
+To locate your development API key, login to the [Trigger.dev
+dashboard](https://cloud.trigger.dev) and select the Project you want to
+connect to. Then click on the Environments & API Keys tab in the left menu.
+You can copy your development API Key from the field at the top of this page.
+(Your development key will start with `tr_dev_`).
+
+## Adding Environment Variables
+
+Create a `.env` file at the root of your project and include your Trigger API key and URL like this:
+
+```bash
+TRIGGER_API_KEY=ENTER_YOUR_DEVELOPMENT_API_KEY_HERE
+TRIGGER_API_URL=https://api.trigger.dev # this is only necessary if you are self-hosting
+```
+
+Replace `ENTER_YOUR_DEVELOPMENT_API_KEY_HERE` with the actual API key obtained from the previous step.
+
+## Syncing Environment Variable types (TypeScript)
+
+You will have type errors for your environment variables unless you run this command:
+
+```sh
+npx svelte-kit sync
+```
+
+## Configuring the Trigger Client
+
+Create a file at `<root>/src/trigger.ts` or `<root>/trigger.ts` depending on whether you're using the `src` directory or not. `<root>` represents the root directory of your project.
+
+Next, add the following code to the file which creates and exports a new `TriggerClient`:
+
+```typescript src/trigger.(ts/js)
+// trigger.ts (for TypeScript) or trigger.js (for JavaScript)
+
+import { TriggerClient } from "@trigger.dev/sdk";
+import { TRIGGER_API_KEY, TRIGGER_API_URL } from "$env/static/private";
+
+export const client = new TriggerClient({
+  id: "my-app",
+  apiKey: TRIGGER_API_KEY,
+  apiUrl: TRIGGER_API_URL,
+});
+```
+
+Replace **"my-app"** with an appropriate identifier for your project.
+
+## Creating the API Route
+
+To establish an API route for interacting with Trigger.dev, follow these steps based on your project's file type and structure
+
+Create a new file named `+server.(ts/js)` within the `src/routes/api/trigger` directory, and add the following code:
+
+```typescript
+import { createSvelteRoute } from "@trigger.dev/sveltekit";
+import { client } from "../../../trigger";
+
+//import all jobs
+import "../../../jobs";
+
+// Create the Svelte route handler using the createSvelteRoute function
+const svelteRoute = createSvelteRoute(client);
+
+// Define your API route handler
+export const POST = svelteRoute.POST;
+```
+
+## Creating the Example Job
+
+1. Create a folder named `jobs` alongside your `src` directory
+2. Inside the `jobs` folder, add two files named `example.(ts/js)` and `index.(ts/js)`.
+
+<CodeGroup>
+
+```typescript src/jobs/example.(ts/js)
+import { eventTrigger } from "@trigger.dev/sdk";
+import { client } from "../trigger";
+
+// your first job
+client.defineJob({
+  id: "example-job",
+  name: "Example Job",
+  version: "0.0.1",
+  trigger: eventTrigger({
+    name: "example.event",
+  }),
+  run: async (payload, io, ctx) => {
+    await io.logger.info("Hello world!", { payload });
+
+    return {
+      message: "Hello world!",
+    };
+  },
+});
+```
+
+```typescript src/jobs/index.(ts/js)
+// export all your job files here
+export * from "./example";
+```
+
+</CodeGroup>
+
+## Additonal Job Definitions
+
+You can define more job definitions by creating additional files in the `jobs` folder and exporting them in the `src/jobs/index` file.
+
+For example, in `index.(ts/js)`, you can export other job files like this:
+
+```typescript
+// export all your job files here
+export * from "./example";
+export * from "./other-job-file";
+```
+
+## Adding Configuration to `package.json`
+
+Inside the `package.json` file, add the following configuration under the root object:
+
+```json
+"trigger.dev": {
+  "endpointId": "my-app"
+}
+```
+
+Your `package.json` file might look something like this:
+
+```json
+{
+  "name": "my-app",
+  "version": "1.0.0",
+  "dependencies": {
+    // ... other dependencies
+  },
+  "trigger.dev": {
+    "endpointId": "my-app"
+  }
+}
+```
+
+Replace **"my-app"** with the appropriate identifier you used during the step for creating the Trigger Client.
+
+## Running
+
+### Run your Sveltekit app
+
+Run your Sveltekit app locally. You need to use the `--host` flag to allow the Trigger.dev CLI to connect to your app.
+
+For example:
+
+<CodeGroup>
+
+```bash npm
+npm run dev -- --open --host
+```
+
+```bash pnpm
+pnpm run dev -- --open --host
+```
+
+```bash yarn
+yarn run dev -- --open --host
+```
+
+</CodeGroup>
+
+### Run the CLI 'dev' command
+
+In a **_separate terminal window or tab_** run:
+
+<CodeGroup>
+
+```bash npm
+npx @trigger.dev/cli@latest dev --port 5173
+```
+
+```bash pnpm
+pnpm dlx @trigger.dev/cli@latest dev --port 5173
+```
+
+```bash yarn
+yarn dlx @trigger.dev/cli@latest dev --port 5173
+```
+
+</CodeGroup>
+<br />
+<Note>
+  You can optionally pass the port if you're not running on 3000 by adding
+  `--port 5173` to the end
+</Note>
+<Note>
+  You can optionally pass the hostname if you're not running on localhost by adding
+  `--hostname <host>`. Example, in case your Sveltekit app is running on 0.0.0.0: `--hostname 0.0.0.0`.
+</Note>

docs/_snippets/stable-key-param.mdx

@@ -1,4 +1,4 @@
-<ResponseField name="key" type="string" required>
-  Should be a stable and unique key inside the `run()`. See
+<ResponseField name="cacheKey" type="string" required>
+  Should be a stable and unique cache key inside the `run()`. See
   [resumability](/documentation/concepts/resumability) for more information.
 </ResponseField>

docs/documentation/concepts/client-adaptors.mdx

@@ -5,7 +5,7 @@ description: "The Client is how you interact with the API, through an Adaptor."
 
 ## Client
 
-A Client is used to connect to a specific [Project](/documentation/concepts/projects) by using an [API Key](/documentation/concepts/environments-apikeys).
+A Client is used to connect to a specific [Project](/documentation/concepts/projects) by using an [API Key](/documentation/concepts/environments-endpoints).
 
 Clients are created using the `TriggerClient` class.
 
@@ -24,11 +24,12 @@ Adaptors allows Clients to receive data from the Trigger API. They do this by cr
 
 Each platform has one or more adaptors, see the guides below:
 
-| Platform                                          | Adaptor              |
-| ------------------------------------------------- | -------------------- |
-| [Next.js](/documentation/guides/platforms/nextjs) | `createPagesRoute()` |
-| [Next.js](/documentation/guides/platforms/nextjs) | `createAppRoute()`   |
-| [NestJS](/documentation/guides/manual/nestjs)     | `TriggerDevModule`   |
-| [Astro](/documentation/guides/platforms/astro)    | `createAstroRoute()` |
-| [Remix](/documentation/guides/platforms/remix)    | `createRemixRoute()` |
-| Express                                           | Coming soon          |
+| Platform                                               | Adaptor               |
+| ------------------------------------------------------ | --------------------- |
+| [Next.js](/documentation/guides/platforms/nextjs)      | `createPagesRoute()`  |
+| [Next.js](/documentation/guides/platforms/nextjs)      | `createAppRoute()`    |
+| [NestJS](/documentation/guides/manual/nestjs)          | `TriggerDevModule`    |
+| [Astro](/documentation/guides/platforms/astro)         | `createAstroRoute()`  |
+| [Remix](/documentation/guides/platforms/remix)         | `createRemixRoute()`  |
+| [Sveltekit](/documentation/guides/platforms/sveltekit) | `createSvelteRoute()` |
+| Express                                                | Coming soon           |

docs/documentation/concepts/limitations.mdx

@@ -1,31 +0,0 @@
----
-title: "Limitations"
----
-
-There are a few limitations that are important to understand.
-
-In the latest version:
-
-- Runs on localhost are limited to 5 minutes.
-- On long-running servers (not serverless) Runs can be retried erroneously.
-- Compute intensive jobs are not well supported.
-
-## Runs on localhost are limited to 5 minutes
-
-When developing locally the [CLI dev command](/documentation/guides/cli#dev-command) uses [ngrok](https://ngrok.com/) so messages can be sent to your machine.
-
-Ngrok has a timeout of 5 minutes on a Request/Response cycle. so, if a localhost Run takes longer than 5 minutes to complete, the Run will fail.
-
-This limitation will be removed in the future by adding an alternative run strategy that works well on localhost and long-running servers. This won't use the request/response cycle.
-
-## On long-running servers (not serverless) Runs can be retried erroneously
-
-Currently the only way that Runs are performed is by a Request/Response cycle when `run` is called on a Job. This is optimized for serverless functions (where you have to use a Request/Response cycle), but not for long-running servers.
-
-This limitation will be removed in the future by adding an alternative mode so Jobs works well on localhost and long-running servers. This won't use the request/response cycle.
-
-## Compute intensive jobs are not well supported
-
-Currently the only way that Runs are performed is inside a Request/Response cycle when `run` is called on a Job. This is not a good way to perform compute intensive jobs.
-
-In the future we will add good support for compute intensive jobs.