Fixes #611: intervalTriggers of >10 never starting

- Better login page title and subtitle
- Better title and new ‘having issues’ panel on the magic link pages

.changeset/chilled-plants-exercise.md

@@ -0,0 +1,14 @@
+---
+"@trigger.dev/integration-kit": minor
+"@trigger.dev/eslint-plugin": minor
+"@trigger.dev/sdk": minor
+"@trigger.dev/sveltekit": minor
+"@trigger.dev/express": minor
+"@trigger.dev/nestjs": minor
+"@trigger.dev/nextjs": minor
+"@trigger.dev/astro": minor
+"@trigger.dev/core": minor
+"@trigger.dev/cli": minor
+---
+
+Drop support for Node v16, require Node >= 18. This allows us to use native fetch in our SDK which paves the way for multi-platform support.

.changeset/cool-stingrays-invent.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/cli": patch
+---
+
+allow users to update trigger-dev packages to a specific version

.changeset/neat-feet-wait.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sveltekit": patch
+---
+
+SvelteKit adaptor package

.changeset/seven-turkeys-worry.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/cli": patch
+---
+
+When indexing user's jobs errors are now stored and displayed

.changeset/sharp-lobsters-cross.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/resend": patch
+"@trigger.dev/core": patch
+---
+
+Allow task property values to be blank, but strip them out before persisting them

.changeset/tidy-toys-turn.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/cli": patch
+---
+
+allow CLI dev to use injected environment variable

.changeset/tiny-candles-type.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/cli": patch
+---
+
+remove unused envFile param in sendEvent.ts cmd

.github/workflows/e2e.yml

@@ -0,0 +1,65 @@
+name: "🧪 E2E Tests"
+on:
+  workflow_call:
+jobs:
+  e2e:
+    name: "🧪 E2E Tests"
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    steps:
+      - name: 🐳 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 7.18
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: |
+          # Setup environment variables
+          cp ./.env.example ./.env
+          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
+
+          # Build packages
+          pnpm run build --filter @references/nextjs-test^...
+          pnpm --filter @trigger.dev/database generate
+
+          # Move trigger-cli bin to correct place
+          pnpm install --frozen-lockfile
+
+          # Execute tests
+          pnpm run docker
+          pnpm run db:migrate
+          pnpm run db:seed
+          pnpm run test:e2e
+
+          # Cleanup
+          pnpm run docker:stop
+
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.github/workflows/pr_checks.yml

@@ -0,0 +1,31 @@
+name: 🤖 PR Checks
+
+on:
+  pull_request_target:
+    branches:
+      - main
+    paths-ignore:
+      - "**.md"
+      - ".github/CODEOWNERS"
+      - ".github/ISSUE_TEMPLATE/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  typecheck:
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
+
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
+
+  e2e:
+    uses: ./.github/workflows/e2e.yml
+    secrets: inherit

.github/workflows/publish.yml

@@ -38,125 +38,19 @@ env:
 
 jobs:
   typecheck:
-    name: ʦ TypeScript
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
 
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: 📀 Generate Prisma Client
-        run: pnpm run generate
-
-      - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
-
-  unitTests:
-    name: Unit Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Run Unit Tests
-        run: |
-          pnpm run test
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
 
   e2e:
-    name: e2e Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
-
-      - name: Run Playwright tests
-        run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
-
-          # Build packages
-          pnpm run build --filter @references/nextjs-test^...
-          pnpm --filter @trigger.dev/database generate
-
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+    uses: ./.github/workflows/e2e.yml
+    secrets: inherit
 
   publish:
-    needs: [typecheck, unitTests, e2e]
+    needs: [typecheck, units, e2e]
     runs-on: buildjet-4vcpu-ubuntu-2204
     outputs:
       version: ${{ steps.get_version.outputs.version }}

.github/workflows/release.yml

@@ -4,16 +4,11 @@ on:
   push:
     branches:
       - main
-    paths:
-      - ".github/workflows/release.yml"
-      - "packages/**"
-      - "!packages/**/*.md"
-      - ".changeset/**"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "pnpm-lock.yaml"
-      - "pnpm-workspace.yaml"
-      - "turbo.json"
+    paths-ignore:
+      - "**.md"
+      - ".github/CODEOWNERS"
+      - ".github/ISSUE_TEMPLATE/**"
+    
 
 jobs:
   release:

.github/workflows/typecheck.yml

@@ -0,0 +1,32 @@
+name: "ʦ TypeScript"
+on:
+  workflow_call:
+jobs:
+  typecheck:
+    runs-on: buildjet-4vcpu-ubuntu-2204
+
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 7.18
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔎 Type check
+        run: pnpm run typecheck --filter webapp

.github/workflows/unit-tests.yml

@@ -0,0 +1,30 @@
+name: "🧪 Unit Tests"
+on:
+  workflow_call:
+jobs:
+  unitTests:
+    name: "🧪 Unit Tests"
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 7.18
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Run Unit Tests
+        run: |
+          pnpm run test

.nvmrc

@@ -1 +1 @@
-v18.12.1
+v18.18.0

CHANGESETS.md

@@ -32,7 +32,26 @@ Please follow the best-practice of adding changesets in the same commit as the c
 
 !MAKE SURE TO UPDATE THE TAG IN THE INSTRUCTIONS BELOW!
 
-1. Add changesets as usual `pnpm run changeset:add`
-2. Create a snapshot version (replace "dev" with your tag) `pnpm exec changeset version --snapshot dev`
-3. Build the packages: `pnpm run build --filter "@trigger.dev/*"`
-4. Publish the snapshot (replace "dev" with your tag) `pnpm exec changeset publish --no-git-tag --snapshot --tag dev`
+1. Add changesets as usual
+
+```sh
+pnpm run changeset:add
+```
+
+2. Create a snapshot version (replace "prerelease" with your tag)
+
+```sh
+pnpm exec changeset version --snapshot prerelease
+```
+
+3. Build the packages:
+
+```sh
+pnpm run build --filter "@trigger.dev/*"
+```
+
+4. Publish the snapshot (replace "dev" with your tag)
+
+```sh
+pnpm exec changeset publish --no-git-tag --snapshot --tag prerelease
+```

CONTRIBUTING.md

@@ -52,9 +52,13 @@ branch are tagged into a release monthly.
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
 6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+
    ```
    pnpm run docker
    ```
+
+   This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
+
 7. Migrate the database
    ```
    pnpm run db:migrate

README.md

@@ -14,6 +14,18 @@
 
 </div>
 
+# ✨🎃 Get involved with Hacktoberfest 2023! 🎃✨
+
+All of October we're participating in Hacktoberfest and invite you to join us! We have a bunch of issues labeled `🎃 Hacktoberfest` that are ready for you to work on which will count towards Hacktoberfest. We are also running our own game, earn 💎 points to win swag!
+
+- Check out our [Hacktoberfest landing page](https://trigger.dev/hacktoberfest) for how to participate and win swag.
+- Contribute to either our [/trigger.dev](https://github.com/triggerdotdev/trigger.dev/labels/%F0%9F%8E%83%20hacktoberfest) or [/jobs-showcase](https://github.com/triggerdotdev/jobs-showcase/labels/%F0%9F%8E%83%20hacktoberfest) repositories and complete issues marked `🎃 Hacktoberfest` to be eligible for swag.
+- Join our [Discord](https://discord.gg/JtBAxBr2m3) and get involved in with the community.
+
+_New to Hacktober? Check out the [Hacktoberfest website](https://hacktoberfest.digitalocean.com/) for more information._
+
+🎃 **Happy Hacking!** 🎃
+
 # About Trigger.dev
 
 Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
@@ -62,8 +74,8 @@ Click the links to join the discussions about our upcoming features.
 | Dashboard                                                                            | View every Task in every Run                        | ✅     |
 | Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
 | React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
+| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
 | [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [React frameworks](https://github.com/triggerdotdev/trigger.dev/discussions/411)     | Support for Remix, Astro, RedwoodJS & more          | 🛠️     |
 | [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
 | Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
 | Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
@@ -83,3 +95,9 @@ We provide an official trigger.dev docker image you can use to easily self-host
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
+
+## 🙏 to our contributors
+
+<a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />
+</a>

apps/webapp/app/components/SetupCommands.tsx

@@ -44,28 +44,7 @@ export function InitCommand({ appOrigin, apiKey }: { appOrigin: string; apiKey:
   );
 }
 
-export function RunDevCommand() {
-  return (
-    <ClientTabs defaultValue="npm">
-      <ClientTabsList>
-        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
-        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
-        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
-      </ClientTabsList>
-      <ClientTabsContent value={"npm"}>
-        <ClipboardField variant="primary/medium" className="mb-4" value={`npm run dev`} />
-      </ClientTabsContent>
-      <ClientTabsContent value={"pnpm"}>
-        <ClipboardField variant="primary/medium" className="mb-4" value={`pnpm run dev`} />
-      </ClientTabsContent>
-      <ClientTabsContent value={"yarn"}>
-        <ClipboardField variant="primary/medium" className="mb-4" value={`yarn run dev`} />
-      </ClientTabsContent>
-    </ClientTabs>
-  );
-}
-
-export function TriggerDevCommand() {
+export function RunDevCommand({ extra }: { extra?: string }) {
   return (
     <ClientTabs defaultValue="npm">
       <ClientTabsList>
@@ -77,34 +56,67 @@ export function TriggerDevCommand() {
         <ClipboardField
           variant="primary/medium"
           className="mb-4"
-          value={`npx @trigger.dev/cli@latest dev`}
+          value={`npm run dev${extra ? ` ${extra}` : ""}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"pnpm"}>
         <ClipboardField
           variant="primary/medium"
           className="mb-4"
-          value={`pnpm dlx @trigger.dev/cli@latest dev`}
+          value={`pnpm run dev${extra ? ` ${extra}` : ""}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"yarn"}>
         <ClipboardField
           variant="primary/medium"
           className="mb-4"
-          value={`yarn dlx @trigger.dev/cli@latest dev`}
+          value={`yarn run dev${extra ? ` ${extra}` : ""}`}
         />
       </ClientTabsContent>
     </ClientTabs>
   );
 }
 
-export function TriggerDevStep() {
+export function TriggerDevCommand({ extra }: { extra?: string }) {
+  return (
+    <ClientTabs defaultValue="npm">
+      <ClientTabsList>
+        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
+      </ClientTabsList>
+      <ClientTabsContent value={"npm"}>
+        <ClipboardField
+          variant="primary/medium"
+          className="mb-4"
+          value={`npx @trigger.dev/cli@latest dev${extra ? ` ${extra}` : ""}`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"pnpm"}>
+        <ClipboardField
+          variant="primary/medium"
+          className="mb-4"
+          value={`pnpm dlx @trigger.dev/cli@latest dev${extra ? ` ${extra}` : ""}`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"yarn"}>
+        <ClipboardField
+          variant="primary/medium"
+          className="mb-4"
+          value={`yarn dlx @trigger.dev/cli@latest dev${extra ? ` ${extra}` : ""}`}
+        />
+      </ClientTabsContent>
+    </ClientTabs>
+  );
+}
+
+export function TriggerDevStep({ extra }: { extra?: string }) {
   return (
     <>
       <Paragraph spacing>
         In a <span className="text-amber-400">separate terminal window or tab</span> run:
       </Paragraph>
-      <TriggerDevCommand />
+      <TriggerDevCommand extra={extra} />
       <Paragraph spacing variant="small">
         If you’re not running on the default you can specify the port by adding{" "}
         <InlineCode variant="extra-small">--port 3001</InlineCode> to the end.

apps/webapp/app/components/environments/EndpointIndexStatus.tsx

@@ -0,0 +1,74 @@
+import { CheckCircleIcon, ClockIcon, XCircleIcon } from "@heroicons/react/20/solid";
+import { EndpointIndexStatus } from "@trigger.dev/database";
+import { cn } from "~/utils/cn";
+import { Spinner } from "../primitives/Spinner";
+
+export function EndpointIndexStatusIcon({ status }: { status: EndpointIndexStatus }) {
+  switch (status) {
+    case "PENDING":
+      return <ClockIcon className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />;
+    case "STARTED":
+      return <Spinner className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />;
+    case "SUCCESS":
+      return (
+        <CheckCircleIcon className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />
+      );
+    case "FAILURE":
+      return <XCircleIcon className={cn("h-4 w-4", endpointIndexStatusClassNameColor(status))} />;
+  }
+}
+
+export function EndpointIndexStatusLabel({ status }: { status: EndpointIndexStatus }) {
+  switch (status) {
+    case "PENDING":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+    case "STARTED":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+    case "SUCCESS":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+    case "FAILURE":
+      return (
+        <span className={endpointIndexStatusClassNameColor(status)}>
+          {endpointIndexStatusTitle(status)}
+        </span>
+      );
+  }
+}
+
+export function endpointIndexStatusTitle(status: EndpointIndexStatus): string {
+  switch (status) {
+    case "PENDING":
+      return "Pending";
+    case "STARTED":
+      return "Started";
+    case "SUCCESS":
+      return "Success";
+    case "FAILURE":
+      return "Failure";
+  }
+}
+
+export function endpointIndexStatusClassNameColor(status: EndpointIndexStatus): string {
+  switch (status) {
+    case "PENDING":
+      return "text-dimmed";
+    case "STARTED":
+      return "text-blue-500";
+    case "SUCCESS":
+      return "text-green-500";
+    case "FAILURE":
+      return "text-rose-500";
+  }
+}

apps/webapp/app/components/frameworks/FrameworkSelector.tsx

@@ -66,7 +66,7 @@ export function FrameworkSelector() {
           <FrameworkLink to={projectSetupNuxtPath(organization, project)}>
             <NuxtLogo className="w-32" />
           </FrameworkLink>
-          <FrameworkLink to={projectSetupSvelteKitPath(organization, project)}>
+          <FrameworkLink to={projectSetupSvelteKitPath(organization, project)} supported>
             <SvelteKitLogo className="w-44" />
           </FrameworkLink>
           <FrameworkLink to={projectSetupFastifyPath(organization, project)}>

apps/webapp/app/components/primitives/Callout.tsx

@@ -13,6 +13,7 @@ import {
 import { Link } from "@remix-run/react";
 import { cn } from "~/utils/cn";
 import { Paragraph } from "./Paragraph";
+import { Spinner } from "./Spinner";
 
 export const variantClasses = {
   info: {
@@ -51,8 +52,16 @@ export const variantClasses = {
     textColor: "text-blue-200",
     linkClassName: "transition hover:bg-blue-400/40",
   },
+  pending: {
+    className: "border-blue-400/20 bg-blue-800/30",
+    icon: <Spinner className="h-5 w-5 shrink-0 " />,
+    textColor: "text-blue-300",
+    linkClassName: "transition hover:bg-blue-400/40",
+  },
 } as const;
 
+export type CalloutVariant = keyof typeof variantClasses;
+
 export function Callout({
   children,
   className,
@@ -63,7 +72,7 @@ export function Callout({
   children?: React.ReactNode;
   className?: string;
   icon?: React.ReactNode;
-  variant: keyof typeof variantClasses;
+  variant: CalloutVariant;
   to?: string;
 }) {
   const variantDefinition = variantClasses[variant];

apps/webapp/app/components/primitives/FormError.tsx

@@ -2,8 +2,17 @@ import type { z } from "zod";
 import { Paragraph } from "./Paragraph";
 import { NamedIcon } from "./NamedIcon";
 import { motion } from "framer-motion";
+import { cn } from "~/utils/cn";
 
-export function FormError({ children, id }: { children: React.ReactNode; id?: string }) {
+export function FormError({
+  children,
+  id,
+  className,
+}: {
+  children: React.ReactNode;
+  id?: string;
+  className?: string;
+}) {
   return (
     <>
       {children && (
@@ -11,7 +20,7 @@ export function FormError({ children, id }: { children: React.ReactNode; id?: st
           initial={{ opacity: 0 }}
           animate={{ opacity: 1 }}
           transition={{ duration: 0.3 }}
-          className="flex items-start gap-0.5"
+          className={cn("flex items-start gap-0.5", className)}
         >
           <NamedIcon name="error" className="h-4 w-4 shrink-0 justify-start" />
           <Paragraph id={id} variant="extra-small" className="text-rose-500">

apps/webapp/app/components/primitives/Switch.tsx

@@ -15,7 +15,7 @@ const variations = {
     container: "flex items-center gap-x-1.5 rounded hover:bg-slate-850 pr-1 py-[0.1rem] pl-1.5",
     root: "h-3 w-6",
     thumb: "h-2.5 w-2.5 data-[state=checked]:translate-x-2.5 data-[state=unchecked]:translate-x-0",
-    text: "text-xs text-slate-400 group-hover:text-slate-200 mt-0.5",
+    text: "text-xs text-slate-400 group-hover:text-slate-200 hover:cursor-pointer",
   },
 };
 

apps/webapp/app/components/primitives/Toast.tsx

@@ -1,7 +1,7 @@
 import { ExclamationCircleIcon, XMarkIcon } from "@heroicons/react/20/solid";
 import { CheckCircleIcon } from "@heroicons/react/24/solid";
-import { AnimatePresence, motion } from "framer-motion";
-import toast, { Toaster, resolveValue, useToasterStore } from "react-hot-toast";
+import { Toaster, toast } from "sonner";
+
 import { useTypedLoaderData } from "remix-typedjson";
 import { loader } from "~/root";
 import { useEffect } from "react";
@@ -11,79 +11,55 @@ const permanentToastDuration = 60 * 60 * 24 * 1000;
 
 export function Toast() {
   const { toastMessage } = useTypedLoaderData<typeof loader>();
-
   useEffect(() => {
     if (!toastMessage) {
       return;
     }
     const { message, type, options } = toastMessage;
 
-    switch (type) {
-      case "success":
-        toast.success(message, {
-          duration: options.ephemeral ? defaultToastDuration : permanentToastDuration,
-        });
-        break;
-      case "error":
-        toast.error(message, {
-          duration: options.ephemeral ? defaultToastDuration : permanentToastDuration,
-        });
-        break;
-      default:
-        throw new Error(`${type} is not handled`);
-    }
+    toast.custom((t) => <ToastUI variant={type} message={message} t={t as string} />, {
+      duration: options.ephemeral ? defaultToastDuration : permanentToastDuration,
+    });
   }, [toastMessage]);
 
+  return <Toaster />;
+}
+
+export function ToastUI({
+  variant,
+  message,
+  t,
+  toastWidth = 356, // Default width, matches what sonner provides by default
+}: {
+  variant: "error" | "success";
+  message: string;
+  t: string;
+  toastWidth?: string | number;
+}) {
   return (
-    <Toaster
-      position="bottom-right"
-      toastOptions={{
-        success: {
-          icon: <CheckCircleIcon className="h-6 w-6 text-green-600" />,
-        },
-        error: {
-          icon: <ExclamationCircleIcon className="h-6 w-6 text-rose-600" />,
-        },
+    <div
+      className={`self-end rounded-lg border border-slate-750 bg-midnight-900 shadow-md`}
+      style={{
+        width: toastWidth,
       }}
     >
-      {(t) => (
-        <AnimatePresence>
-          <motion.div
-            className="flex gap-2 rounded-lg border border-slate-750 bg-no-repeat p-4 text-bright shadow-md"
-            style={{
-              opacity: t.visible ? 1 : 0,
-              background:
-                "radial-gradient(at top, hsla(271, 91%, 65%, 0.18), hsla(221, 83%, 53%, 0.18)) hsla(221, 83%, 53%, 0.18)",
-            }}
-            initial={{ opacity: 0, y: 100 }}
-            animate={t.visible ? "visible" : "hidden"}
-            variants={{
-              hidden: {
-                opacity: 0,
-                y: 0,
-                transition: {
-                  duration: 0.15,
-                  ease: "easeInOut",
-                },
-              },
-              visible: {
-                opacity: 1,
-                y: 0,
-                transition: {
-                  duration: 0.3,
-                  ease: "easeInOut",
-                },
-              },
-            }}
-          >
-            {t.icon}
-            {resolveValue(t.message, t)}
-            <button className="p-1" onClick={() => toast.dismiss(t.id)}>
-              <XMarkIcon className="h-4 w-4 text-bright" />
-            </button>
-          </motion.div>
-        </AnimatePresence>
-      )}
-    </Toaster>
+      <div
+        className="flex w-full gap-2 rounded-lg bg-no-repeat p-4 text-bright"
+        style={{
+          background:
+            "radial-gradient(at top, hsla(271, 91%, 65%, 0.18), hsla(221, 83%, 53%, 0.18)) hsla(221, 83%, 53%, 0.18)",
+        }}
+      >
+        {variant === "success" ? (
+          <CheckCircleIcon className="h-6 w-6 text-green-600" />
+        ) : (
+          <ExclamationCircleIcon className="h-6 w-6 text-rose-600" />
+        )}
+        {message}
+        <button className="ms-auto p-1" onClick={() => toast.dismiss(t)}>
+          <XMarkIcon className="h-4 w-4 text-bright" />
+        </button>
+      </div>
+    </div>
   );
 }

apps/webapp/app/components/stories/ToastUI.stories.tsx

@@ -0,0 +1,47 @@
+import type { Meta, StoryObj } from "@storybook/react";
+import { Toaster, toast } from "sonner";
+import { ToastUI } from "../primitives/Toast";
+import { Button } from "../primitives/Buttons";
+
+const meta: Meta = {
+  title: "Primitives/Toast",
+};
+
+export default meta;
+
+type Story = StoryObj<typeof Collection>;
+
+export const Toasts: Story = {
+  render: () => <Collection />,
+};
+
+function Collection() {
+  return (
+    <div className="flex flex-col items-start gap-y-4 p-4">
+      <ToastUI variant="success" message="Success UI" t="-" />
+      <ToastUI variant="error" message="Error UI" t="-" />
+      <br />
+      <Button
+        variant="primary/large"
+        onClick={() =>
+          toast.custom((t) => <ToastUI variant="success" message="Success" t={t as string} />, {
+            duration: Infinity, // Prevents auto-dismissal for demo purposes
+          })
+        }
+      >
+        Success
+      </Button>
+      <Button
+        variant="primary/large"
+        onClick={() =>
+          toast.custom((t) => <ToastUI variant="error" message="Error" t={t as string} />, {
+            duration: Infinity,
+          })
+        }
+      >
+        Error
+      </Button>
+      <Toaster />
+    </div>
+  );
+}

apps/webapp/app/hooks/useFilterJobs.ts

@@ -1,9 +1,21 @@
 import { ProjectJob } from "./useJobs";
 import { useTextFilter } from "./useTextFilter";
+import { useToggleFilter } from "./useToggleFilter";
 
-export function useFilterJobs(jobs: ProjectJob[]) {
-  const { filterText, setFilterText, filteredItems } = useTextFilter<ProjectJob>({
+export function useFilterJobs(jobs: ProjectJob[], onlyActiveJobs = false) {
+  const toggleFilterRes = useToggleFilter<ProjectJob>({
     items: jobs,
+    filter: (job, onlyActiveJobs) => {
+      if (onlyActiveJobs && job.status !== "ACTIVE") {
+        return false;
+      }
+      return true;
+    },
+    defaultValue: onlyActiveJobs,
+  });
+
+  const textFilterRes = useTextFilter<ProjectJob>({
+    items: toggleFilterRes.filteredItems,
     filter: (job, text) => {
       if (job.slug.toLowerCase().includes(text.toLowerCase())) return true;
       if (job.title.toLowerCase().includes(text.toLowerCase())) return true;
@@ -24,5 +36,11 @@ export function useFilterJobs(jobs: ProjectJob[]) {
     },
   });
 
-  return { filterText, setFilterText, filteredItems };
+  return {
+    filteredItems: textFilterRes.filteredItems,
+    filterText: textFilterRes.filterText,
+    setFilterText: textFilterRes.setFilterText,
+    onlyActiveJobs: toggleFilterRes.isToggleActive,
+    setOnlyActiveJobs: toggleFilterRes.setToggleActive,
+  };
 }

apps/webapp/app/hooks/useToggleFilter.ts

@@ -0,0 +1,21 @@
+import { useMemo, useState } from "react";
+
+type ToggleFilterProps<T> = {
+  items: T[];
+  filter: (item: T, isToggleActive: boolean) => boolean;
+  defaultValue?: boolean;
+};
+
+export function useToggleFilter<T>({ items, filter, defaultValue = false }: ToggleFilterProps<T>) {
+  const [isToggleActive, setToggleActive] = useState(defaultValue);
+
+  const filteredItems = useMemo<T[]>(() => {
+    return items.filter((item) => filter(item, isToggleActive));
+  }, [items, isToggleActive]);
+
+  return {
+    isToggleActive,
+    setToggleActive,
+    filteredItems,
+  };
+}

apps/webapp/app/models/indexEndpoint.server.ts

@@ -1,14 +0,0 @@
-import { z } from "zod";
-
-const IndexEndpointStatsSchema = z.object({
-  jobs: z.number(),
-  sources: z.number(),
-  dynamicTriggers: z.number(),
-  dynamicSchedules: z.number(),
-});
-
-export type IndexEndpointStats = z.infer<typeof IndexEndpointStatsSchema>;
-
-export function parseEndpointIndexStats(stats: unknown): IndexEndpointStats {
-  return IndexEndpointStatsSchema.parse(stats);
-}

apps/webapp/app/platform/zodWorker.server.ts

@@ -14,7 +14,7 @@ import { run as graphileRun, parseCronItems } from "graphile-worker";
 import omit from "lodash.omit";
 import { z } from "zod";
 import { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
-import { logger } from "~/services/logger.server";
+import { workerLogger as logger } from "~/services/logger.server";
 
 export interface MessageCatalogSchema {
   [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
@@ -81,6 +81,18 @@ export type ZodWorkerDequeueOptions = {
   tx?: PrismaClientOrTransaction;
 };
 
+const CLEANUP_TASK_NAME = "__cleanupOldJobs";
+const REPORTER_TASK_NAME = "__reporter";
+
+export type ZodWorkerCleanupOptions = {
+  frequencyExpression: string; // cron expression
+  ttl: number;
+  maxCount: number;
+  taskOptions?: CronItemOptions;
+};
+
+type ZodWorkerReporter = (event: string, properties: Record<string, any>) => Promise<void>;
+
 export type ZodWorkerOptions<TMessageCatalog extends MessageCatalogSchema> = {
   name: string;
   runnerOptions: RunnerOptions;
@@ -88,6 +100,8 @@ export type ZodWorkerOptions<TMessageCatalog extends MessageCatalogSchema> = {
   schema: TMessageCatalog;
   tasks: ZodTasks<TMessageCatalog>;
   recurringTasks?: ZodRecurringTasks;
+  cleanup?: ZodWorkerCleanupOptions;
+  reporter?: ZodWorkerReporter;
 };
 
 export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
@@ -98,6 +112,8 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
   #tasks: ZodTasks<TMessageCatalog>;
   #recurringTasks?: ZodRecurringTasks;
   #runner?: GraphileRunner;
+  #cleanup: ZodWorkerCleanupOptions | undefined;
+  #reporter?: ZodWorkerReporter;
 
   constructor(options: ZodWorkerOptions<TMessageCatalog>) {
     this.#name = options.name;
@@ -106,6 +122,8 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     this.#runnerOptions = options.runnerOptions;
     this.#tasks = options.tasks;
     this.#recurringTasks = options.recurringTasks;
+    this.#cleanup = options.cleanup;
+    this.#reporter = options.reporter;
   }
 
   get graphileWorkerSchema() {
@@ -337,12 +355,45 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       taskList[key] = task;
     }
 
+    if (this.#cleanup) {
+      const task: Task = (payload, helpers) => {
+        return this.#handleCleanup(payload, helpers);
+      };
+
+      taskList[CLEANUP_TASK_NAME] = task;
+    }
+
+    if (this.#reporter) {
+      const task: Task = (payload, helpers) => {
+        return this.#handleReporter(payload, helpers);
+      };
+
+      taskList[REPORTER_TASK_NAME] = task;
+    }
+
     return taskList;
   }
 
   #createCronItemsFromRecurringTasks() {
     const cronItems: CronItem[] = [];
 
+    if (this.#cleanup) {
+      cronItems.push({
+        pattern: this.#cleanup.frequencyExpression,
+        identifier: CLEANUP_TASK_NAME,
+        task: CLEANUP_TASK_NAME,
+        options: this.#cleanup.taskOptions,
+      });
+    }
+
+    if (this.#reporter) {
+      cronItems.push({
+        pattern: "50 * * * *", // Every hour at 50 minutes past the hour
+        identifier: REPORTER_TASK_NAME,
+        task: REPORTER_TASK_NAME,
+      });
+    }
+
     if (!this.#recurringTasks) {
       return cronItems;
     }
@@ -434,6 +485,112 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     }
   }
 
+  async #handleCleanup(rawPayload: unknown, helpers: JobHelpers): Promise<void> {
+    if (!this.#cleanup) {
+      return;
+    }
+
+    const job = helpers.job;
+
+    logger.debug("Received cleanup task", {
+      payload: rawPayload,
+      job,
+    });
+
+    const parsedPayload = RawCronPayloadSchema.safeParse(rawPayload);
+
+    if (!parsedPayload.success) {
+      throw new Error(
+        `Failed to parse cleanup task payload: ${JSON.stringify(parsedPayload.error)}`
+      );
+    }
+
+    const payload = parsedPayload.data;
+
+    // Add the this.#cleanup.ttl to the payload._cron.ts
+    const expirationDate = new Date(payload._cron.ts.getTime() - this.#cleanup.ttl);
+
+    logger.debug("Cleaning up old jobs", {
+      expirationDate,
+      payload,
+    });
+
+    const rawResults = await this.#prisma.$queryRawUnsafe(
+      `WITH rows AS (SELECT id FROM ${this.graphileWorkerSchema}.jobs WHERE run_at < $1 AND locked_at IS NULL AND max_attempts = attempts LIMIT $2 FOR UPDATE) DELETE FROM ${this.graphileWorkerSchema}.jobs WHERE id IN (SELECT id FROM rows) RETURNING id`,
+      expirationDate,
+      this.#cleanup.maxCount
+    );
+
+    const results = Array.isArray(rawResults) ? rawResults : [];
+
+    logger.debug("Cleaned up old jobs", {
+      count: results.length,
+      expirationDate,
+      payload,
+    });
+
+    if (this.#reporter) {
+      await this.#reporter("cleanup_stats", {
+        count: results.length,
+        expirationDate,
+        ts: payload._cron.ts,
+      });
+    }
+  }
+
+  async #handleReporter(rawPayload: unknown, helpers: JobHelpers): Promise<void> {
+    if (!this.#reporter) {
+      return;
+    }
+
+    logger.debug("Received reporter task", {
+      payload: rawPayload,
+    });
+
+    const parsedPayload = RawCronPayloadSchema.safeParse(rawPayload);
+
+    if (!parsedPayload.success) {
+      throw new Error(
+        `Failed to parse cleanup task payload: ${JSON.stringify(parsedPayload.error)}`
+      );
+    }
+
+    const payload = parsedPayload.data;
+
+    // Subtract an hour from the payload._cron.ts
+    const startAt = new Date(payload._cron.ts.getTime() - 1000 * 60 * 60);
+
+    const schema = z.array(z.object({ count: z.coerce.number() }));
+
+    // Count the number of jobs that have been added since the startAt date and before the payload._cron.ts date
+    const rawAddedResults = await this.#prisma.$queryRawUnsafe(
+      `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs WHERE created_at > $1 AND created_at < $2`,
+      startAt,
+      payload._cron.ts
+    );
+
+    const addedCountResults = schema.parse(rawAddedResults)[0];
+
+    // Count the total number of jobs in the jobs table
+    const rawTotalResults = await this.#prisma.$queryRawUnsafe(
+      `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs`
+    );
+
+    const totalCountResults = schema.parse(rawTotalResults)[0];
+
+    logger.debug("Calculated metrics about the jobs table", {
+      rawAddedResults,
+      rawTotalResults,
+      payload,
+    });
+
+    await this.#reporter("queue_metrics", {
+      addedCount: addedCountResults.count,
+      totalCount: totalCountResults.count,
+      ts: payload._cron.ts,
+    });
+  }
+
   #logDebug(message: string, args?: any) {
     logger.debug(`[worker][${this.#name}] ${message}`, args);
   }

apps/webapp/app/presenters/EnvironmentsPresenter.server.ts

@@ -1,13 +1,19 @@
 import { PrismaClient, prisma } from "~/db.server";
-import { IndexEndpointStats, parseEndpointIndexStats } from "~/models/indexEndpoint.server";
 import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
 import type {
   Endpoint,
   EndpointIndex,
+  EndpointIndexStatus,
   RuntimeEnvironment,
   RuntimeEnvironmentType,
 } from "@trigger.dev/database";
+import {
+  EndpointIndexError,
+  EndpointIndexErrorSchema,
+  IndexEndpointStats,
+  parseEndpointIndexStats,
+} from "@trigger.dev/core";
 
 export type Client = {
   slug: string;
@@ -34,9 +40,11 @@ export type ClientEndpoint =
       url: string;
       indexWebhookPath: string;
       latestIndex?: {
+        status: EndpointIndexStatus;
         source: string;
         updatedAt: Date;
-        stats: IndexEndpointStats;
+        stats?: IndexEndpointStats;
+        error?: EndpointIndexError;
       };
       environment: {
         id: string;
@@ -81,9 +89,11 @@ export class EnvironmentsPresenter {
             indexingHookIdentifier: true,
             indexings: {
               select: {
+                status: true,
                 source: true,
                 updatedAt: true,
                 stats: true,
+                error: true,
               },
               take: 1,
               orderBy: {
@@ -214,7 +224,7 @@ const environmentSortOrder: RuntimeEnvironmentType[] = [
 
 function endpointClient(
   endpoint: Pick<Endpoint, "id" | "slug" | "url" | "indexingHookIdentifier"> & {
-    indexings: Pick<EndpointIndex, "source" | "updatedAt" | "stats">[];
+    indexings: Pick<EndpointIndex, "status" | "source" | "updatedAt" | "stats" | "error">[];
   },
   environment: Pick<RuntimeEnvironment, "id" | "apiKey" | "type">,
   baseUrl: string
@@ -227,9 +237,13 @@ function endpointClient(
     indexWebhookPath: `${baseUrl}/api/v1/endpoints/${environment.id}/${endpoint.slug}/index/${endpoint.indexingHookIdentifier}`,
     latestIndex: endpoint.indexings[0]
       ? {
+          status: endpoint.indexings[0].status,
           source: endpoint.indexings[0].source,
           updatedAt: endpoint.indexings[0].updatedAt,
           stats: parseEndpointIndexStats(endpoint.indexings[0].stats),
+          error: endpoint.indexings[0].error
+            ? EndpointIndexErrorSchema.parse(endpoint.indexings[0].error)
+            : undefined,
         }
       : undefined,
     environment: environment,

apps/webapp/app/presenters/OrgUsagePresenter.server.ts

@@ -1,4 +1,5 @@
 import { PrismaClient, prisma } from "~/db.server";
+import { logger } from "~/services/logger.server";
 
 export class OrgUsagePresenter {
   #prismaClient: PrismaClient;
@@ -33,6 +34,7 @@ export class OrgUsagePresenter {
         createdAt: {
           gte: new Date(new Date().getFullYear(), new Date().getMonth(), 1),
         },
+        internal: false,
       },
     });
 
@@ -44,6 +46,7 @@ export class OrgUsagePresenter {
           gte: startOfLastMonth,
           lt: startOfMonth,
         },
+        internal: false,
       },
     });
 
@@ -63,7 +66,7 @@ export class OrgUsagePresenter {
         month: string;
         count: number;
       }[]
-    >`SELECT TO_CHAR("createdAt", 'YYYY-MM') as month, COUNT(*) as count FROM "JobRun" WHERE "organizationId" = ${organization.id} AND "createdAt" >= NOW() - INTERVAL '6 months' GROUP BY month ORDER BY month ASC`;
+    >`SELECT TO_CHAR("createdAt", 'YYYY-MM') as month, COUNT(*) as count FROM "JobRun" WHERE "organizationId" = ${organization.id} AND "createdAt" >= NOW() - INTERVAL '6 months' AND "internal" = FALSE GROUP BY month ORDER BY month ASC`;
 
     const chartData = chartDataRaw.map((obj) => ({
       name: obj.month,
@@ -139,11 +142,13 @@ export class OrgUsagePresenter {
       },
     });
 
+    const chartDataDisplay = fillInMissingMonthlyData(chartData, 6);
+
     return {
       id: organization.id,
       runsCount,
       runsCountLastMonth,
-      chartData: fillInMissingMonthlyData(chartData, 6),
+      chartData: chartDataDisplay,
       totalJobs,
       totalJobsLastMonth,
       totalIntegrations,
@@ -166,7 +171,7 @@ function fillInMissingMonthlyData(
 
   const startMonth = new Date(
     new Date(currentMonth).getFullYear(),
-    new Date(currentMonth).getMonth() - totalNumberOfMonths,
+    new Date(currentMonth).getMonth() - (totalNumberOfMonths - 2),
     1
   )
     .toISOString()
@@ -182,17 +187,36 @@ function fillInMissingMonthlyData(
   return completeData;
 }
 
+// Start month will be like 2023-03 and endMonth will be like 2023-10
+// The result should be an array of months between these two months, including the start and end month
+// So for example, if startMonth is 2023-03 and endMonth is 2023-10, the result should be:
+// ["2023-03", "2023-04", "2023-05", "2023-06", "2023-07", "2023-08", "2023-09", "2023-10"]
 function getMonthsBetween(startMonth: string, endMonth: string): string[] {
-  const startDate = new Date(startMonth);
-  const endDate = new Date(endMonth);
+  // Initialize result array
+  const result: string[] = [];
 
-  const months = [];
-  let currentDate = startDate;
+  // Parse the year and month from startMonth and endMonth
+  let [startYear, startMonthNum] = startMonth.split("-").map(Number);
+  let [endYear, endMonthNum] = endMonth.split("-").map(Number);
 
-  while (currentDate <= endDate) {
-    months.push(currentDate.toISOString().slice(0, 7));
-    currentDate = new Date(currentDate.setMonth(currentDate.getMonth() + 1));
+  // Loop through each month between startMonth and endMonth
+  for (let year = startYear; year <= endYear; year++) {
+    let monthStart = year === startYear ? startMonthNum : 1;
+    let monthEnd = year === endYear ? endMonthNum : 12;
+
+    for (let month = monthStart; month <= monthEnd; month++) {
+      // Format the month into a string and add it to the result array
+      result.push(`${year}-${String(month).padStart(2, "0")}`);
+    }
   }
 
-  return months;
+  return result;
+}
+
+function getLastSecondOfMonth(endMonth: string) {
+  const [year, month] = endMonth.split("-").map(Number);
+  const nextMonthFirstDay = new Date(year, month, 1);
+  nextMonthFirstDay.setDate(0);
+  nextMonthFirstDay.setHours(23, 59, 59);
+  return nextMonthFirstDay;
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam._index/route.tsx

@@ -19,6 +19,7 @@ import {
   PageTitleRow,
 } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Switch } from "~/components/primitives/Switch";
 import { TextLink } from "~/components/primitives/TextLink";
 import { useFilterJobs } from "~/hooks/useFilterJobs";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -62,8 +63,11 @@ export default function Page() {
   const organization = useOrganization();
   const project = useProject();
   const { jobs } = useTypedLoaderData<typeof loader>();
-  const { filterText, setFilterText, filteredItems } = useFilterJobs(jobs);
-  const hasJobs = jobs.length > 0;
+  const { filterText, setFilterText, filteredItems, onlyActiveJobs, setOnlyActiveJobs } =
+    useFilterJobs(jobs);
+  const totalJobs = jobs.length;
+  const hasJobs = totalJobs > 0;
+  const activeJobCount = jobs.filter((j) => j.status === "ACTIVE").length;
 
   return (
     <PageContainer className={hasJobs ? "" : "grid-rows-1"}>
@@ -74,7 +78,8 @@ export default function Page() {
           </PageTitleRow>
           <PageInfoRow>
             <PageInfoGroup>
-              <PageInfoProperty icon={"job"} label={"Active Jobs"} value={jobs.length} />
+              <PageInfoProperty icon={"job"} label={"All Jobs"} value={totalJobs} />
+              <PageInfoProperty icon={"job"} label={"Active Jobs"} value={activeJobCount} />
             </PageInfoGroup>
           </PageInfoRow>
         </PageHeader>
@@ -96,7 +101,7 @@ export default function Page() {
                       </Callout>
                     )}
                     <div className="mb-2 flex flex-col">
-                      <div className="flex w-full">
+                      <div className="flex w-full gap-x-2">
                         <Input
                           placeholder="Search Jobs"
                           variant="tertiary"
@@ -106,6 +111,13 @@ export default function Page() {
                           onChange={(e) => setFilterText(e.target.value)}
                           autoFocus
                         />
+                        <Switch
+                          variant="small"
+                          label="Active Jobs"
+                          checked={onlyActiveJobs}
+                          onCheckedChange={setOnlyActiveJobs}
+                          className={"shrink-0"}
+                        />
                         <HelpTrigger title="Example Jobs and inspiration" />
                       </div>
                     </div>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx

@@ -6,7 +6,7 @@ import { useEventSource } from "remix-utils";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { Button } from "~/components/primitives/Buttons";
-import { Callout } from "~/components/primitives/Callout";
+import { Callout, CalloutVariant } from "~/components/primitives/Callout";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DateTime } from "~/components/primitives/DateTime";
 import { FormError } from "~/components/primitives/FormError";
@@ -18,8 +18,14 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { Sheet, SheetBody, SheetContent, SheetHeader } from "~/components/primitives/Sheet";
 import { ClientEndpoint } from "~/presenters/EnvironmentsPresenter.server";
 import { endpointStreamingPath } from "~/utils/pathBuilder";
-import { RuntimeEnvironmentType } from "../../../../../packages/database/src";
+import { EndpointIndexStatus, RuntimeEnvironmentType } from "../../../../../packages/database/src";
 import { bodySchema } from "../resources.environments.$environmentParam.endpoint";
+import {
+  EndpointIndexStatusIcon,
+  EndpointIndexStatusLabel,
+  endpointIndexStatusTitle,
+} from "~/components/environments/EndpointIndexStatus";
+import { CodeBlock } from "~/components/code/CodeBlock";
 
 type ConfigureEndpointSheetProps = {
   slug: string;
@@ -119,15 +125,29 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd
                   method="post"
                   action={`/resources/environments/${endpoint.environment.id}/endpoint/${endpoint.id}`}
                 >
-                  <Callout variant="success" className="justiy-between items-center">
-                    <Paragraph variant="small" className="grow text-green-200">
-                      Endpoint configured. Last refreshed:{" "}
-                      {endpoint.latestIndex ? (
-                        <DateTime date={endpoint.latestIndex.updatedAt} />
-                      ) : (
-                        "–"
-                      )}
-                    </Paragraph>
+                  <Callout
+                    variant="info"
+                    icon={
+                      <EndpointIndexStatusIcon status={endpoint.latestIndex?.status ?? "PENDING"} />
+                    }
+                    className="justiy-between items-center"
+                  >
+                    <div className="flex grow items-center gap-2">
+                      <EndpointIndexStatusLabel
+                        status={endpoint.latestIndex?.status ?? "PENDING"}
+                      />
+                      <Paragraph variant="small" className="grow">
+                        Last refreshed:{" "}
+                        {endpoint.latestIndex ? (
+                          <>
+                            <DateTime date={endpoint.latestIndex.updatedAt} />
+                          </>
+                        ) : (
+                          "–"
+                        )}
+                      </Paragraph>
+                    </div>
+
                     <Button
                       variant="primary/small"
                       type="submit"
@@ -138,6 +158,11 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd
                       {refreshingEndpoint ? "Refreshing" : "Refresh now"}
                     </Button>
                   </Callout>
+                  {endpoint.latestIndex?.error && (
+                    <FormError className="p-2">
+                      <pre>{endpoint.latestIndex.error.message}</pre>
+                    </FormError>
+                  )}
                 </refreshEndpointFetcher.Form>
               </div>
               <div className="max-w-full overflow-hidden">
@@ -155,3 +180,16 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd
     </Sheet>
   );
 }
+
+function calloutVariantFromStatus(status: EndpointIndexStatus): CalloutVariant {
+  switch (status) {
+    case "PENDING":
+      return "pending";
+    case "STARTED":
+      return "pending";
+    case "SUCCESS":
+      return "success";
+    case "FAILURE":
+      return "error";
+  }
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx

@@ -3,11 +3,16 @@ import { LoaderArgs } from "@remix-run/server-runtime";
 import { useEffect, useMemo, useState } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { useEventSource } from "remix-utils";
+import {
+  EndpointIndexStatusIcon,
+  EndpointIndexStatusLabel,
+} from "~/components/environments/EndpointIndexStatus";
 import { EnvironmentLabel, environmentTitle } from "~/components/environments/EnvironmentLabel";
 import { HowToUseApiKeysAndEndpoints } from "~/components/helpContent/HelpContentText";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { BreadcrumbLink } from "~/components/navigation/NavBar";
-import { Button, ButtonContent } from "~/components/primitives/Buttons";
+import { Badge } from "~/components/primitives/Badge";
+import { ButtonContent } from "~/components/primitives/Buttons";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DateTime } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
@@ -38,7 +43,6 @@ import { ProjectParamSchema, projectEnvironmentsStreamingPath } from "~/utils/pa
 import { requestUrl } from "~/utils/requestUrl.server";
 import { RuntimeEnvironmentType } from "../../../../../packages/database/src";
 import { ConfigureEndpointSheet } from "./ConfigureEndpointSheet";
-import { Badge } from "~/components/primitives/Badge";
 import { FirstEndpointSheet } from "./FirstEndpointSheet";
 
 export const loader = async ({ request, params }: LoaderArgs) => {
@@ -180,6 +184,7 @@ export default function Page() {
                               <TableHeaderCell>Environment</TableHeaderCell>
                               <TableHeaderCell>Url</TableHeaderCell>
                               <TableHeaderCell>Last refreshed</TableHeaderCell>
+                              <TableHeaderCell>Last refresh Status</TableHeaderCell>
                               <TableHeaderCell>Jobs</TableHeaderCell>
                               <TableHeaderCell hiddenLabel>Go to page</TableHeaderCell>
                             </TableRow>
@@ -268,7 +273,7 @@ function EndpointRow({
               <EnvironmentLabel environment={{ type }} />
             </div>
           </TableCell>
-          <TableCell onClick={onClick} colSpan={4} alignment="right">
+          <TableCell onClick={onClick} colSpan={5} alignment="right">
             <div className="flex items-center justify-end gap-4">
               <span className="text-amber-500">
                 The {environmentTitle({ type })} environment is not configured
@@ -290,7 +295,17 @@ function EndpointRow({
           <TableCell onClick={onClick}>
             {endpoint.latestIndex ? <DateTime date={endpoint.latestIndex.updatedAt} /> : "–"}
           </TableCell>
-          <TableCell onClick={onClick}>{endpoint.latestIndex?.stats.jobs ?? "–"}</TableCell>
+          <TableCell onClick={onClick}>
+            {endpoint.latestIndex ? (
+              <div className="flex items-center gap-1">
+                <EndpointIndexStatusIcon status={endpoint.latestIndex.status} />
+                <EndpointIndexStatusLabel status={endpoint.latestIndex.status} />
+              </div>
+            ) : (
+              "–"
+            )}
+          </TableCell>
+          <TableCell onClick={onClick}>{endpoint.latestIndex?.stats?.jobs ?? "–"}</TableCell>
           <TableCellChevron onClick={onClick} />
         </TableRow>
       );

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.integrations/route.tsx

@@ -152,7 +152,7 @@ function PossibleIntegrationsList({
           onCheckedChange={setOnlyShowIntegrations}
           variant="small"
           label={
-            <span className="inline-flex items-center gap-1">
+            <span className="mt-0.5 inline-flex items-center gap-1">
               <IntegrationIcon /> Trigger.dev Integrations
             </span>
           }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.integrations_.$clientParam._index/route.tsx

@@ -8,6 +8,7 @@ import { Callout } from "~/components/primitives/Callout";
 import { Header2 } from "~/components/primitives/Headers";
 import { Help, HelpContent, HelpTrigger } from "~/components/primitives/Help";
 import { Input } from "~/components/primitives/Input";
+import { Paragraph } from "~/components/primitives/Paragraph";
 import { useFilterJobs } from "~/hooks/useFilterJobs";
 import { useIntegrationClient } from "~/hooks/useIntegrationClient";
 import { JobListPresenter } from "~/presenters/JobListPresenter.server";
@@ -52,10 +53,8 @@ export default function Page() {
       {(open) => (
         <div className={cn("grid h-full gap-4", open ? "grid-cols-2" : "grid-cols-1")}>
           <div className="grow">
-            <div className="mb-2 flex items-center justify-between gap-x-2">
-              {jobs.length === 0 ? (
-                <Header2>Jobs using this integration will appear here</Header2>
-              ) : (
+            <div className="mb-2 flex items-center justify-end gap-x-2">
+              {jobs.length !== 0 && (
                 <Input
                   placeholder="Search Jobs"
                   variant="tertiary"
@@ -68,9 +67,9 @@ export default function Page() {
               <HelpTrigger title="How do I use this integration?" />
             </div>
             {jobs.length === 0 ? (
-              <>
-                <JobSkeleton />
-              </>
+              <div className="mt-8 rounded border border-border px-2 py-6 text-center">
+                <Paragraph variant="small">Jobs using this Integration will appear here.</Paragraph>
+              </div>
             ) : (
               <JobsTable
                 jobs={filteredItems}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.test/route.tsx

@@ -35,6 +35,7 @@ import { TestJobService } from "~/services/jobs/testJob.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { Handle } from "~/utils/handle";
+import { isValidIcon } from "~/utils/icon";
 import { JobParamsSchema, jobRunDashboardPath, trimTrailingSlash } from "~/utils/pathBuilder";
 
 export const loader = async ({ request, params }: LoaderArgs) => {
@@ -228,7 +229,7 @@ export default function Page() {
                       }}
                     >
                       <DetailCell
-                        leadingIcon={example.icon ?? CodeBracketIcon}
+                        leadingIcon={isValidIcon(example.icon) ? example.icon : CodeBracketIcon}
                         leadingIconClassName="text-blue-500"
                         label={example.name}
                         trailingIcon={example.id === selectedCodeSampleId ? "check" : "plus"}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.setup.sveltekit/route.tsx

@@ -1,23 +1,113 @@
-import { SvelteKitLogo } from "~/assets/logos/SveltekitLogo";
-import { FrameworkComingSoon } from "~/components/frameworks/FrameworkComingSoon";
+import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
+import invariant from "tiny-invariant";
+import { Feedback } from "~/components/Feedback";
+import { PageGradient } from "~/components/PageGradient";
+import { RunDevCommand, TriggerDevStep } from "~/components/SetupCommands";
+import { StepContentContainer } from "~/components/StepContentContainer";
+import { InlineCode } from "~/components/code/InlineCode";
 import { BreadcrumbLink } from "~/components/navigation/NavBar";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { ClipboardField } from "~/components/primitives/ClipboardField";
+import { Header1 } from "~/components/primitives/Headers";
+import { NamedIcon } from "~/components/primitives/NamedIcon";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { StepNumber } from "~/components/primitives/StepNumber";
+import { useAppOrigin } from "~/hooks/useAppOrigin";
+import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
+import { useDevEnvironment } from "~/hooks/useEnvironments";
+import { useOrganization } from "~/hooks/useOrganizations";
+import { useProject } from "~/hooks/useProject";
 import { Handle } from "~/utils/handle";
-import { trimTrailingSlash } from "~/utils/pathBuilder";
-
+import { projectSetupPath, trimTrailingSlash } from "~/utils/pathBuilder";
+import { Callout } from "~/components/primitives/Callout";
+import { Badge } from "~/components/primitives/Badge";
 export const handle: Handle = {
   breadcrumb: (match) => (
     <BreadcrumbLink to={trimTrailingSlash(match.pathname)} title="SvelteKit" />
   ),
 };
 
-export default function Page() {
+export default function SetUpSveltekit() {
+  const organization = useOrganization();
+  const project = useProject();
+  useProjectSetupComplete();
+  const devEnvironment = useDevEnvironment();
+  invariant(devEnvironment, "Dev environment must be defined");
   return (
-    <FrameworkComingSoon
-      frameworkName="SvelteKit"
-      githubIssueUrl="https://github.com/triggerdotdev/trigger.dev/issues/453"
-      githubIssueNumber={453}
-    >
-      <SvelteKitLogo className="w-56" />
-    </FrameworkComingSoon>
+    <PageGradient>
+      <div className="mx-auto max-w-3xl">
+        <div className="flex items-center justify-between">
+          <Header1 spacing className="text-bright">
+            Get setup in 5 minutes
+          </Header1>
+          <div className="flex items-center gap-2">
+            <LinkButton
+              to={projectSetupPath(organization, project)}
+              variant="tertiary/small"
+              LeadingIcon={Squares2X2Icon}
+            >
+              Choose a different framework
+            </LinkButton>
+            <Feedback
+              button={
+                <Button variant="tertiary/small" LeadingIcon={ChatBubbleLeftRightIcon}>
+                  I'm stuck!
+                </Button>
+              }
+              defaultValue="help"
+            />
+          </div>
+        </div>
+        <div>
+          <Callout
+            variant={"info"}
+            to="https://github.com/triggerdotdev/trigger.dev/discussions/430"
+            className="mb-8"
+          >
+            Trigger.dev has full support for serverless. We will be adding support for long-running
+            servers soon.
+          </Callout>
+          <div>
+            <StepNumber
+              stepNumber="1"
+              title="Follow the steps from the Sveltekit manual installation guide"
+            />
+            <StepContentContainer className="flex flex-col gap-2">
+              <Paragraph className="mt-2">Copy your server API Key to your clipboard:</Paragraph>
+              <div className="mb-2 flex w-full items-center justify-between">
+                <ClipboardField
+                  secure
+                  className="w-fit"
+                  value={devEnvironment.apiKey}
+                  variant={"secondary/medium"}
+                  icon={<Badge variant="outline">Server</Badge>}
+                />
+              </div>
+              <Paragraph>Now follow this guide:</Paragraph>
+              <LinkButton
+                to="https://trigger.dev/docs/documentation/guides/manual/sveltekit"
+                variant="primary/medium"
+                TrailingIcon="external-link"
+              >
+                Manual installation guide
+              </LinkButton>
+              <div className="flex items-start justify-start gap-2"></div>
+            </StepContentContainer>
+            <StepNumber stepNumber="2" title="Run your sveltekit app" />
+            <StepContentContainer>
+              <RunDevCommand extra=" -- --open --host" />
+            </StepContentContainer>
+            <StepNumber stepNumber="3" title="Run the CLI 'dev' command" />
+            <StepContentContainer>
+              <TriggerDevStep extra=" --port 5173" />
+            </StepContentContainer>
+            <StepNumber stepNumber="6" title="Wait for Jobs" displaySpinner />
+            <StepContentContainer>
+              <Paragraph>This page will automatically refresh.</Paragraph>
+            </StepContentContainer>
+          </div>
+        </div>
+      </div>
+    </PageGradient>
   );
 }

apps/webapp/app/routes/api.v1.endpointindex.$indexId.ts

@@ -0,0 +1,68 @@
+import { ActionArgs, json } from "@remix-run/server-runtime";
+import {
+  EndpointIndexErrorSchema,
+  GetEndpointIndexResponse,
+  GetEndpointIndexResponseSchema,
+} from "@trigger.dev/core";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { IndexEndpointService } from "~/services/endpoints/indexEndpoint.server";
+import { logger } from "~/services/logger.server";
+
+const ParamsSchema = z.object({
+  indexId: z.string(),
+});
+
+export async function loader({ request, params }: ActionArgs) {
+  if (request.method.toUpperCase() !== "GET") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+  if (!authenticationResult) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  const { indexId } = parsedParams.data;
+
+  const endpointIndex = await prisma.endpointIndex.findUnique({
+    where: {
+      id: indexId,
+      endpoint: {
+        environmentId: authenticatedEnv.id,
+      },
+    },
+  });
+
+  if (!endpointIndex) {
+    logger.info("EndpointIndex not found", { url: request.url });
+    return json({ error: "EndpointIndex not found" }, { status: 404 });
+  }
+
+  const parsed = GetEndpointIndexResponseSchema.safeParse(endpointIndex);
+
+  if (!parsed.success) {
+    logger.info("EndpointIndex failed parsing", { errors: parsed.error.issues, endpointIndex });
+    const parseFailResult: GetEndpointIndexResponse = {
+      status: "FAILURE",
+      error: {
+        message: "Invalid endpoint index",
+      },
+      updatedAt: new Date(),
+    };
+    return json(parseFailResult, { status: 500 });
+  }
+
+  return json(parsed.data);
+}

apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier.ts

@@ -1,7 +1,7 @@
 import { ActionArgs, LoaderArgs, json } from "@remix-run/server-runtime";
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { z } from "zod";
-import { PrismaClient, prisma } from "~/db.server";
+import { $transaction, PrismaClient, prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { workerQueue } from "~/services/worker.server";
 import { safeJsonParse } from "~/utils/json";
@@ -93,43 +93,53 @@ export class TriggerEndpointIndexHookService {
       body,
     });
 
-    const endpoint = await this.#prismaClient.endpoint.findUnique({
-      where: {
-        environmentId_slug: {
-          environmentId,
-          slug: endpointSlug,
+    await $transaction(this.#prismaClient, async (tx) => {
+      const endpoint = await tx.endpoint.findUnique({
+        where: {
+          environmentId_slug: {
+            environmentId,
+            slug: endpointSlug,
+          },
         },
-      },
-      include: {
-        environment: true,
-      },
-    });
+        include: {
+          environment: true,
+        },
+      });
 
-    if (!endpoint) {
-      throw new Error("Endpoint not found");
-    }
-
-    if (endpoint.indexingHookIdentifier !== indexHookIdentifier) {
-      throw new Error("Index hook identifier is invalid");
-    }
-
-    const reason = parseReasonFromBody(body);
-
-    // Index the endpoint in 5 seconds from now
-    await workerQueue.enqueue(
-      "indexEndpoint",
-      {
-        id: endpoint.id,
-        source: "HOOK",
-        reason,
-        sourceData: body,
-      },
-      {
-        runAt: new Date(Date.now() + 5000),
-        maxAttempts:
-          endpoint.environment.type === RuntimeEnvironmentType.DEVELOPMENT ? 1 : undefined,
+      if (!endpoint) {
+        throw new Error("Endpoint not found");
       }
-    );
+
+      if (endpoint.indexingHookIdentifier !== indexHookIdentifier) {
+        throw new Error("Index hook identifier is invalid");
+      }
+
+      const reason = parseReasonFromBody(body);
+
+      const index = await tx.endpointIndex.create({
+        data: {
+          endpointId: endpoint.id,
+          status: "PENDING",
+          source: "HOOK",
+          reason,
+          sourceData: body,
+        },
+      });
+
+      // Index the endpoint in 5 seconds from now
+      await workerQueue.enqueue(
+        "performEndpointIndexing",
+        {
+          id: index.id,
+        },
+        {
+          runAt: new Date(Date.now() + 5000),
+          maxAttempts:
+            endpoint.environment.type === RuntimeEnvironmentType.DEVELOPMENT ? 1 : undefined,
+          tx,
+        }
+      );
+    });
   }
 }
 

apps/webapp/app/routes/api.v1.runs.$runId.tasks.ts

@@ -280,7 +280,7 @@ export class RunTaskService {
           noop: taskBody.noop,
           delayUntil: taskBody.delayUntil,
           params: taskBody.params ?? undefined,
-          properties: taskBody.properties ?? undefined,
+          properties: this.#filterProperties(taskBody.properties) ?? undefined,
           redact: taskBody.redact ?? undefined,
           operation: taskBody.operation,
           callbackUrl,
@@ -305,7 +305,7 @@ export class RunTaskService {
           {
             id: task.id,
           },
-          { tx, runAt: task.delayUntil ?? undefined }
+          { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
         );
       } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
         if (taskBody.callback.timeoutInSeconds > 0) {
@@ -325,4 +325,14 @@ export class RunTaskService {
 
     return task ? taskWithAttemptsToServerTask(task) : undefined;
   }
+
+  #filterProperties(properties: RunTaskBodyOutput["properties"]): RunTaskBodyOutput["properties"] {
+    if (!properties) return;
+
+    return properties.filter((property) => {
+      if (!property) return false;
+
+      return typeof property.label === "string" && typeof property.text === "string";
+    });
+  }
 }

apps/webapp/app/routes/api.v1.sources.http.$id.ts

@@ -6,11 +6,24 @@ import { HandleHttpSourceService } from "~/services/sources/handleHttpSource.ser
 export async function action({ request, params }: ActionArgs) {
   logger.info("Handling http source", { url: request.url });
 
-  const { id } = z.object({ id: z.string() }).parse(params);
+  try {
+    const { id } = z.object({ id: z.string() }).parse(params);
+    const service = new HandleHttpSourceService();
+    const result = await service.call(id, request);
 
-  const service = new HandleHttpSourceService();
-
-  return await service.call(id, request);
+    return new Response(undefined, {
+      status: result.status,
+    });
+  } catch (e) {
+    if (e instanceof Error) {
+      logger.error("Error handling http source", { error: e.message });
+    } else {
+      logger.error("Error handling http source", { error: JSON.stringify(e) });
+    }
+    return new Response(undefined, {
+      status: 500,
+    });
+  }
 }
 
 export async function loader({ request, params }: LoaderArgs) {

apps/webapp/app/routes/login._index/route.tsx

@@ -60,16 +60,29 @@ export default function LoginPage() {
             <a href="https://trigger.dev">
               <LogoIcon className="mb-4 h-16 w-16" />
             </a>
-            <FormTitle divide={false} title="Log in to Trigger.dev" />
+            <FormTitle divide={false} title="Welcome to Trigger.dev" className="mb-2 pb-0" />
+            <Paragraph variant="small" className="mb-4">
+              Create an account or login
+            </Paragraph>
             <Fieldset>
               <div className="flex flex-col gap-y-2">
                 {data.showGithubAuth && (
-                  <Button type="submit" variant="primary/large" fullWidth>
+                  <Button
+                    type="submit"
+                    variant="primary/large"
+                    fullWidth
+                    data-action="continue with github"
+                  >
                     <NamedIcon name={"github"} className={"mr-1.5 h-4 w-4"} />
                     Continue with GitHub
                   </Button>
                 )}
-                <LinkButton to="/login/magic" variant="secondary/large" fullWidth>
+                <LinkButton
+                  to="/login/magic"
+                  variant="secondary/large"
+                  fullWidth
+                  data-action="continue with email"
+                >
                   <NamedIcon
                     name={"envelope"}
                     className={"mr-1.5 h-4 w-4 text-dimmed transition group-hover:text-bright"}

apps/webapp/app/routes/login.magic/route.tsx

@@ -1,6 +1,6 @@
 import type { ActionArgs, LoaderArgs } from "@remix-run/node";
 import { redirect } from "@remix-run/node";
-import { Form, useTransition } from "@remix-run/react";
+import { Form, useNavigation, useTransition } from "@remix-run/react";
 import { TypedMetaFunction, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { LogoIcon } from "~/components/LogoIcon";
@@ -17,10 +17,10 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { authenticator } from "~/services/auth.server";
 import { commitSession, getUserSession } from "~/services/sessionStorage.server";
 import magicLinkIcon from "./login.magic.svg";
-
 import type { LoaderType as RootLoader } from "~/root";
 import { appEnvTitleTag } from "~/utils";
 import { TextLink } from "~/components/primitives/TextLink";
+import { FormError } from "~/components/primitives/FormError";
 
 export const meta: TypedMetaFunction<typeof loader, { root: RootLoader }> = ({ parentsData }) => ({
   title: `Login to Trigger.dev${appEnvTitleTag(parentsData?.root.appEnv)}`,
@@ -32,10 +32,26 @@ export async function loader({ request }: LoaderArgs) {
   });
 
   const session = await getUserSession(request);
+  const error = session.get("auth:error");
 
-  return typedjson({
-    magicLinkSent: session.has("triggerdotdev:magiclink"),
-  });
+  let magicLinkError: string | undefined;
+  if (error) {
+    if ("message" in error) {
+      magicLinkError = error.message;
+    } else {
+      magicLinkError = JSON.stringify(error, null, 2);
+    }
+  }
+
+  return typedjson(
+    {
+      magicLinkSent: session.has("triggerdotdev:magiclink"),
+      magicLinkError,
+    },
+    {
+      headers: { "Set-Cookie": await commitSession(session) },
+    }
+  );
 }
 
 export async function action({ request }: ActionArgs) {
@@ -50,7 +66,7 @@ export async function action({ request }: ActionArgs) {
     .parse(payload);
 
   if (action === "send") {
-    await authenticator.authenticate("email-link", request, {
+    return authenticator.authenticate("email-link", request, {
       successRedirect: "/login/magic",
       failureRedirect: "/login/magic",
     });
@@ -67,13 +83,13 @@ export async function action({ request }: ActionArgs) {
 }
 
 export default function LoginMagicLinkPage() {
-  const { magicLinkSent } = useTypedLoaderData<typeof loader>();
-  const transition = useTransition();
+  const { magicLinkSent, magicLinkError } = useTypedLoaderData<typeof loader>();
+  const navigate = useNavigation();
 
   const isLoading =
-    (transition.state === "loading" || transition.state === "submitting") &&
-    transition.type === "actionSubmission" &&
-    transition.submission.formData.get("action") === "send";
+    (navigate.state === "loading" || navigate.state === "submitting") &&
+    navigate.formAction !== undefined &&
+    navigate.formData?.get("action") === "send";
 
   return (
     <AppContainer showBackgroundGradient={true}>
@@ -102,12 +118,17 @@ export default function LoginMagicLinkPage() {
                         variant="tertiary/small"
                         LeadingIcon="arrow-left"
                         leadingIconClassName="text-dimmed group-hover:text-bright transition"
+                        data-action="re-enter email"
                       >
                         Re-enter email
                       </Button>
                     }
                     confirmButton={
-                      <LinkButton to="/login" variant="tertiary/small">
+                      <LinkButton
+                        to="/login"
+                        variant="tertiary/small"
+                        data-action="log in using another option"
+                      >
                         Log in using another option
                       </LinkButton>
                     }
@@ -116,7 +137,10 @@ export default function LoginMagicLinkPage() {
               </>
             ) : (
               <>
-                <FormTitle divide={false} title="Log in to Trigger.dev" />
+                <FormTitle divide={false} title="Welcome to Trigger.dev" className="mb-2 pb-0" />
+                <Paragraph variant="small" className="mb-4 text-center">
+                  Create an account or login using your email
+                </Paragraph>
                 <Fieldset className="flex w-full flex-col items-center gap-y-2">
                   <InputGroup>
                     <Label>Your email address</Label>
@@ -137,6 +161,7 @@ export default function LoginMagicLinkPage() {
                     variant="primary/medium"
                     disabled={isLoading}
                     fullWidth
+                    data-action="send a magic link"
                   >
                     <NamedIcon
                       name={isLoading ? "spinner-white" : "envelope"}
@@ -144,6 +169,7 @@ export default function LoginMagicLinkPage() {
                     />
                     {isLoading ? "Sending…" : "Send a magic link"}
                   </Button>
+                  {magicLinkError && <FormError>{magicLinkError}</FormError>}
                 </Fieldset>
                 <Paragraph variant="extra-small" className="my-4 text-center">
                   By logging in with your email you agree to our{" "}
@@ -162,11 +188,28 @@ export default function LoginMagicLinkPage() {
                   variant={"tertiary/small"}
                   LeadingIcon={"arrow-left"}
                   leadingIconClassName="text-dimmed group-hover:text-bright transition"
+                  data-action="all login options"
                 >
                   All login options
                 </LinkButton>
               </>
             )}
+            <div className="mt-8 rounded border border-border px-6 py-4">
+              <Paragraph variant="small" className="mb-2 text-center">
+                Having login issues?
+              </Paragraph>
+              <Paragraph variant="extra-small" className="text-center">
+                Ensure the Magic Link email isn't in your spam folder. If the problem persists,{" "}
+                <TextLink href="mailto:help@trigger.dev" target="_blank">
+                  drop us an email
+                </TextLink>{" "}
+                or let us know on{" "}
+                <TextLink href="https://trigger.dev/discord" target="_blank">
+                  Discord
+                </TextLink>
+                .
+              </Paragraph>
+            </div>
           </div>
         </Form>
       </MainCenteredContainer>

apps/webapp/app/routes/resources.environments.$environmentParam.endpoint.$endpointParam.ts

@@ -1,11 +1,5 @@
-import { parse } from "@conform-to/zod";
 import { ActionArgs, json } from "@remix-run/server-runtime";
 import { z } from "zod";
-import { prisma } from "~/db.server";
-import {
-  CreateEndpointError,
-  CreateEndpointService,
-} from "~/services/endpoints/createEndpoint.server";
 import { IndexEndpointService } from "~/services/endpoints/indexEndpoint.server";
 import { requireUserId } from "~/services/session.server";
 

apps/webapp/app/services/email.server.ts

@@ -1,4 +1,4 @@
-import type { DeliverEmail } from "emails";
+import type { DeliverEmail, SendPlainTextOptions } from "emails";
 import { EmailClient } from "emails";
 import type { SendEmailOptions } from "remix-auth-email-link";
 import { redirect } from "remix-typedjson";
@@ -6,6 +6,7 @@ import { env } from "~/env.server";
 import type { User } from "~/models/user.server";
 import type { AuthUser } from "./authUser";
 import { workerQueue } from "./worker.server";
+import { logger } from "./logger.server";
 
 const client = new EmailClient({
   apikey: env.RESEND_API_KEY,
@@ -20,11 +21,22 @@ export async function sendMagicLinkEmail(options: SendEmailOptions<AuthUser>): P
     throw redirect(options.magicLink);
   }
 
-  return client.send({
-    email: "magic_link",
-    to: options.emailAddress,
-    magicLink: options.magicLink,
-  });
+  logger.debug("Sending magic link email", { emailAddress: options.emailAddress });
+
+  try {
+    return await client.send({
+      email: "magic_link",
+      to: options.emailAddress,
+      magicLink: options.magicLink,
+    });
+  } catch (error) {
+    logger.error("Error sending magic link email", { error: JSON.stringify(error) });
+    throw error;
+  }
+}
+
+export async function sendPlainTextEmail(options: SendPlainTextOptions) {
+  return client.sendPlainText(options);
 }
 
 export async function scheduleWelcomeEmail(user: User) {

apps/webapp/app/services/emailAuth.server.tsx

@@ -5,6 +5,7 @@ import { findOrCreateUser } from "~/models/user.server";
 import { env } from "~/env.server";
 import { sendMagicLinkEmail } from "~/services/email.server";
 import { postAuthentication } from "./postAuth.server";
+import { logger } from "./logger.server";
 
 let secret = env.MAGIC_LINK_SECRET;
 if (!secret) throw new Error("Missing MAGIC_LINK_SECRET env variable.");
@@ -25,6 +26,8 @@ const emailStrategy = new EmailLinkStrategy(
     form: FormData;
     magicLinkVerify: boolean;
   }) => {
+    logger.info("Magic link user authenticated", { email, magicLinkVerify });
+
     try {
       const { user, isNewUser } = await findOrCreateUser({
         email,
@@ -35,6 +38,7 @@ const emailStrategy = new EmailLinkStrategy(
 
       return { userId: user.id };
     } catch (error) {
+      logger.debug("Magic link user failed to authenticate", { error: JSON.stringify(error) });
       throw error;
     }
   }

apps/webapp/app/services/endpointApi.server.ts

@@ -104,6 +104,7 @@ export class EndpointApi {
   }
 
   async indexEndpoint() {
+    const startTimeInMs = performance.now();
     const response = await safeFetch(this.url, {
       method: "POST",
       headers: {
@@ -113,40 +114,13 @@ export class EndpointApi {
       },
     });
 
-    if (!response) {
-      throw new Error(`Could not connect to endpoint ${this.url}`);
-    }
-
-    if (response.status === 401) {
-      const body = await safeBodyFromResponse(response, ErrorWithStackSchema);
-
-      if (body) {
-        return {
-          ok: false,
-          error: body.message,
-        } as const;
-      }
-
-      return {
-        ok: false,
-        error: `Trigger API key is invalid`,
-      } as const;
-    }
-
-    if (!response.ok) {
-      throw new Error(`Could not connect to endpoint ${this.url}. Status code: ${response.status}`);
-    }
-
-    const anyBody = await response.json();
-
-    const data = IndexEndpointResponseSchema.parse(anyBody);
-    const headers = EndpointHeadersSchema.parse(Object.fromEntries(response.headers.entries()));
-
     return {
-      ok: true,
-      data,
-      headers,
-    } as const;
+      response,
+      headerParser: EndpointHeadersSchema,
+      parser: IndexEndpointResponseSchema,
+      errorParser: ErrorWithStackSchema,
+      durationInMs: Math.floor(performance.now() - startTimeInMs),
+    };
   }
 
   async executeJobRequest(options: RunJobBody) {

apps/webapp/app/services/endpoints/createEndpoint.server.ts

@@ -82,12 +82,19 @@ export class CreateEndpointService {
           },
         });
 
+        const endpointIndex = await tx.endpointIndex.create({
+          data: {
+            endpointId: endpoint.id,
+            status: "PENDING",
+            source: "INTERNAL",
+          },
+        });
+
         // Kick off process to fetch the jobs for this endpoint
         await workerQueue.enqueue(
-          "indexEndpoint",
+          "performEndpointIndexing",
           {
-            id: endpoint.id,
-            source: "INTERNAL",
+            id: endpointIndex.id,
           },
           {
             tx,
@@ -96,7 +103,7 @@ export class CreateEndpointService {
           }
         );
 
-        return endpoint;
+        return { ...endpoint, endpointIndex };
       });
 
       return result;

apps/webapp/app/services/endpoints/indexEndpoint.server.ts

@@ -1,23 +1,9 @@
 import type { EndpointIndexSource } from "@trigger.dev/database";
 import { PrismaClient, prisma } from "~/db.server";
-import { findEndpoint } from "~/models/endpoint.server";
-import { EndpointApi } from "../endpointApi.server";
-import { RegisterJobService } from "../jobs/registerJob.server";
-import { logger } from "../logger.server";
-import { RegisterSourceServiceV1 } from "../sources/registerSourceV1.server";
-import { RegisterDynamicScheduleService } from "../triggers/registerDynamicSchedule.server";
-import { RegisterDynamicTriggerService } from "../triggers/registerDynamicTrigger.server";
-import { DisableJobService } from "../jobs/disableJob.server";
-import { RegisterSourceServiceV2 } from "../sources/registerSourceV2.server";
+import { PerformEndpointIndexService } from "./performEndpointIndexService";
 
 export class IndexEndpointService {
   #prismaClient: PrismaClient;
-  #registerJobService = new RegisterJobService();
-  #disableJobService = new DisableJobService();
-  #registerSourceServiceV1 = new RegisterSourceServiceV1();
-  #registerSourceServiceV2 = new RegisterSourceServiceV2();
-  #registerDynamicTriggerService = new RegisterDynamicTriggerService();
-  #registerDynamicScheduleService = new RegisterDynamicScheduleService();
 
   constructor(prismaClient: PrismaClient = prisma) {
     this.#prismaClient = prismaClient;
@@ -29,220 +15,17 @@ export class IndexEndpointService {
     reason?: string,
     sourceData?: any
   ) {
-    const endpoint = await findEndpoint(id);
-
-    // Make a request to the endpoint to fetch a list of jobs
-    const client = new EndpointApi(endpoint.environment.apiKey, endpoint.url);
-
-    const indexResponse = await client.indexEndpoint();
-
-    if (!indexResponse.ok) {
-      throw new Error(indexResponse.error);
-    }
-
-    const { jobs, sources, dynamicTriggers, dynamicSchedules } = indexResponse.data;
-    const { "trigger-version": triggerVersion } = indexResponse.headers;
-
-    logger.debug("Indexing endpoint", {
-      endpointId: endpoint.id,
-      endpointUrl: endpoint.url,
-      endpointSlug: endpoint.slug,
-      source: source,
-      sourceData: sourceData,
-      triggerVersion,
-      stats: {
-        jobs: jobs.length,
-        sources: sources.length,
-        dynamicTriggers: dynamicTriggers.length,
-        dynamicSchedules: dynamicSchedules.length,
-      },
-    });
-
-    if (triggerVersion && triggerVersion !== endpoint.version) {
-      await this.#prismaClient.endpoint.update({
-        where: {
-          id: endpoint.id,
-        },
-        data: {
-          version: triggerVersion,
-        },
-      });
-    }
-
-    const indexStats = {
-      jobs: 0,
-      sources: 0,
-      dynamicTriggers: 0,
-      dynamicSchedules: 0,
-      disabledJobs: 0,
-    };
-
-    const existingJobs = await this.#prismaClient.job.findMany({
-      where: {
-        projectId: endpoint.projectId,
-        deletedAt: null,
-      },
-      include: {
-        aliases: {
-          where: {
-            name: "latest",
-            environmentId: endpoint.environmentId,
-          },
-          include: {
-            version: true,
-          },
-          take: 1,
-        },
-      },
-    });
-
-    for (const job of jobs) {
-      if (!job.enabled) {
-        const disabledJob = await this.#disableJobService
-          .call(endpoint, { slug: job.id, version: job.version })
-          .catch((error) => {
-            logger.error("Failed to disable job", {
-              endpointId: endpoint.id,
-              job,
-              error,
-            });
-
-            return;
-          });
-
-        if (disabledJob) {
-          indexStats.disabledJobs++;
-        }
-      } else {
-        try {
-          const registeredVersion = await this.#registerJobService.call(endpoint, job);
-
-          if (registeredVersion) {
-            indexStats.jobs++;
-          }
-        } catch (error) {
-          logger.error("Failed to register job", {
-            endpointId: endpoint.id,
-            job,
-            error,
-          });
-        }
-      }
-    }
-
-    // TODO: we need to do this for sources, dynamic triggers, and dynamic schedules
-    const missingJobs = existingJobs.filter((job) => {
-      return !jobs.find((j) => j.id === job.slug);
-    });
-
-    if (missingJobs.length > 0) {
-      logger.debug("Disabling missing jobs", {
-        endpointId: endpoint.id,
-        missingJobIds: missingJobs.map((job) => job.slug),
-      });
-
-      for (const job of missingJobs) {
-        const latestVersion = job.aliases[0]?.version;
-
-        if (!latestVersion) {
-          continue;
-        }
-
-        const disabledJob = await this.#disableJobService
-          .call(endpoint, {
-            slug: job.slug,
-            version: latestVersion.version,
-          })
-          .catch((error) => {
-            logger.error("Failed to disable job", {
-              endpointId: endpoint.id,
-              job,
-              error,
-            });
-
-            return;
-          });
-
-        if (disabledJob) {
-          indexStats.disabledJobs++;
-        }
-      }
-    }
-
-    for (const source of sources) {
-      try {
-        switch (source.version) {
-          default:
-          case "1": {
-            await this.#registerSourceServiceV1.call(endpoint, source);
-            break;
-          }
-          case "2": {
-            await this.#registerSourceServiceV2.call(endpoint, source);
-            break;
-          }
-        }
-
-        indexStats.sources++;
-      } catch (error) {
-        logger.error("Failed to register source", {
-          endpointId: endpoint.id,
-          source,
-          error,
-        });
-      }
-    }
-
-    for (const dynamicTrigger of dynamicTriggers) {
-      try {
-        await this.#registerDynamicTriggerService.call(endpoint, dynamicTrigger);
-
-        indexStats.dynamicTriggers++;
-      } catch (error) {
-        logger.error("Failed to register dynamic trigger", {
-          endpointId: endpoint.id,
-          dynamicTrigger,
-          error,
-        });
-      }
-    }
-
-    for (const dynamicSchedule of dynamicSchedules) {
-      try {
-        await this.#registerDynamicScheduleService.call(endpoint, dynamicSchedule);
-
-        indexStats.dynamicSchedules++;
-      } catch (error) {
-        logger.error("Failed to register dynamic schedule", {
-          endpointId: endpoint.id,
-          dynamicSchedule,
-          error,
-        });
-      }
-    }
-
-    logger.debug("Endpoint indexing complete", {
-      endpointId: endpoint.id,
-      indexStats,
-      source,
-      sourceData,
-      reason,
-    });
-
-    return await this.#prismaClient.endpointIndex.create({
+    const endpointIndex = await this.#prismaClient.endpointIndex.create({
       data: {
-        endpointId: endpoint.id,
-        stats: indexStats,
-        data: {
-          jobs,
-          sources,
-          dynamicTriggers,
-          dynamicSchedules,
-        },
+        endpointId: id,
+        status: "PENDING",
         source,
-        sourceData,
         reason,
+        sourceData,
       },
     });
+
+    const performEndpointIndexService = new PerformEndpointIndexService();
+    return await performEndpointIndexService.call(endpointIndex.id);
   }
 }

apps/webapp/app/services/endpoints/performEndpointIndexService.ts

@@ -0,0 +1,338 @@
+import type { EndpointIndexSource } from "@trigger.dev/database";
+import { PrismaClient, prisma } from "~/db.server";
+import { findEndpoint } from "~/models/endpoint.server";
+import { EndpointApi } from "../endpointApi.server";
+import { RegisterJobService } from "../jobs/registerJob.server";
+import { logger } from "../logger.server";
+import { RegisterSourceServiceV1 } from "../sources/registerSourceV1.server";
+import { RegisterDynamicScheduleService } from "../triggers/registerDynamicSchedule.server";
+import { RegisterDynamicTriggerService } from "../triggers/registerDynamicTrigger.server";
+import { DisableJobService } from "../jobs/disableJob.server";
+import { RegisterSourceServiceV2 } from "../sources/registerSourceV2.server";
+import { EndpointIndexError } from "@trigger.dev/core";
+import { safeBodyFromResponse } from "~/utils/json";
+import { fromZodError } from "zod-validation-error";
+import { IndexEndpointStats } from "@trigger.dev/core";
+
+export class PerformEndpointIndexService {
+  #prismaClient: PrismaClient;
+  #registerJobService = new RegisterJobService();
+  #disableJobService = new DisableJobService();
+  #registerSourceServiceV1 = new RegisterSourceServiceV1();
+  #registerSourceServiceV2 = new RegisterSourceServiceV2();
+  #registerDynamicTriggerService = new RegisterDynamicTriggerService();
+  #registerDynamicScheduleService = new RegisterDynamicScheduleService();
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const endpointIndex = await this.#prismaClient.endpointIndex.update({
+      where: {
+        id,
+      },
+      data: {
+        status: "STARTED",
+      },
+      include: {
+        endpoint: {
+          include: {
+            environment: {
+              include: {
+                organization: true,
+                project: true,
+              },
+            },
+          },
+        },
+      },
+    });
+
+    logger.debug("Performing endpoint index", endpointIndex);
+
+    // Make a request to the endpoint to fetch a list of jobs
+    const client = new EndpointApi(
+      endpointIndex.endpoint.environment.apiKey,
+      endpointIndex.endpoint.url
+    );
+    const { response, parser, headerParser, errorParser } = await client.indexEndpoint();
+
+    if (!response) {
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
+      });
+    }
+
+    if (response.status === 401) {
+      const body = await safeBodyFromResponse(response, errorParser);
+
+      if (body) {
+        return updateEndpointIndexWithError(this.#prismaClient, id, {
+          message: body.message,
+        });
+      }
+
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: "Trigger API key is invalid",
+      });
+    }
+
+    if (!response.ok) {
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
+      });
+    }
+
+    const anyBody = await response.json();
+    const bodyResult = parser.safeParse(anyBody);
+
+    if (!bodyResult.success) {
+      const issues: string[] = [];
+      bodyResult.error.issues.forEach((issue) => {
+        if (issue.path.at(0) === "jobs") {
+          const jobIndex = issue.path.at(1) as number;
+          const job = (anyBody as any).jobs[jobIndex];
+
+          if (job) {
+            issues.push(`Job "${job.id}": ${issue.message} at "${issue.path.slice(2).join(".")}".`);
+          }
+        }
+      });
+
+      let friendlyError: string | undefined;
+      if (issues.length > 0) {
+        friendlyError = `Your Jobs have issues:\n${issues.map((issue) => `- ${issue}`).join("\n")}`;
+      } else {
+        friendlyError = fromZodError(bodyResult.error, {
+          prefix: "There's an issue with the format of your Jobs",
+        }).message;
+      }
+
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: friendlyError,
+        raw: bodyResult.error.issues,
+      });
+    }
+
+    const headerResult = headerParser.safeParse(Object.fromEntries(response.headers.entries()));
+    if (!headerResult.success) {
+      const friendlyError = fromZodError(headerResult.error, {
+        prefix: "Your headers are invalid",
+      });
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: friendlyError.message,
+        raw: headerResult.error.issues,
+      });
+    }
+
+    const { jobs, sources, dynamicTriggers, dynamicSchedules } = bodyResult.data;
+    const { "trigger-version": triggerVersion } = headerResult.data;
+    const { endpoint } = endpointIndex;
+
+    if (triggerVersion && triggerVersion !== endpoint.version) {
+      await this.#prismaClient.endpoint.update({
+        where: {
+          id: endpoint.id,
+        },
+        data: {
+          version: triggerVersion,
+        },
+      });
+    }
+
+    const indexStats: IndexEndpointStats = {
+      jobs: 0,
+      sources: 0,
+      dynamicTriggers: 0,
+      dynamicSchedules: 0,
+      disabledJobs: 0,
+    };
+
+    const existingJobs = await this.#prismaClient.job.findMany({
+      where: {
+        projectId: endpoint.projectId,
+        deletedAt: null,
+      },
+      include: {
+        aliases: {
+          where: {
+            name: "latest",
+            environmentId: endpoint.environmentId,
+          },
+          include: {
+            version: true,
+          },
+          take: 1,
+        },
+      },
+    });
+
+    for (const job of jobs) {
+      if (!job.enabled) {
+        const disabledJob = await this.#disableJobService
+          .call(endpoint, { slug: job.id, version: job.version })
+          .catch((error) => {
+            logger.error("Failed to disable job", {
+              endpointId: endpoint.id,
+              job,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledJob) {
+          indexStats.disabledJobs++;
+        }
+      } else {
+        try {
+          const registeredVersion = await this.#registerJobService.call(endpoint, job);
+
+          if (registeredVersion) {
+            if (!job.internal) {
+              indexStats.jobs++;
+            }
+          }
+        } catch (error) {
+          logger.error("Failed to register job", {
+            endpointId: endpoint.id,
+            job,
+            error,
+          });
+        }
+      }
+    }
+
+    // TODO: we need to do this for sources, dynamic triggers, and dynamic schedules
+    const missingJobs = existingJobs.filter((job) => {
+      return !jobs.find((j) => j.id === job.slug);
+    });
+
+    if (missingJobs.length > 0) {
+      logger.debug("Disabling missing jobs", {
+        endpointId: endpoint.id,
+        missingJobIds: missingJobs.map((job) => job.slug),
+      });
+
+      for (const job of missingJobs) {
+        const latestVersion = job.aliases[0]?.version;
+
+        if (!latestVersion) {
+          continue;
+        }
+
+        const disabledJob = await this.#disableJobService
+          .call(endpoint, {
+            slug: job.slug,
+            version: latestVersion.version,
+          })
+          .catch((error) => {
+            logger.error("Failed to disable job", {
+              endpointId: endpoint.id,
+              job,
+              error,
+            });
+
+            return;
+          });
+
+        if (disabledJob) {
+          indexStats.disabledJobs++;
+        }
+      }
+    }
+
+    for (const source of sources) {
+      try {
+        switch (source.version) {
+          default:
+          case "1": {
+            await this.#registerSourceServiceV1.call(endpoint, source);
+            break;
+          }
+          case "2": {
+            await this.#registerSourceServiceV2.call(endpoint, source);
+            break;
+          }
+        }
+
+        indexStats.sources++;
+      } catch (error) {
+        logger.error("Failed to register source", {
+          endpointId: endpoint.id,
+          source,
+          error,
+        });
+      }
+    }
+
+    for (const dynamicTrigger of dynamicTriggers) {
+      try {
+        await this.#registerDynamicTriggerService.call(endpoint, dynamicTrigger);
+
+        indexStats.dynamicTriggers++;
+      } catch (error) {
+        logger.error("Failed to register dynamic trigger", {
+          endpointId: endpoint.id,
+          dynamicTrigger,
+          error,
+        });
+      }
+    }
+
+    for (const dynamicSchedule of dynamicSchedules) {
+      try {
+        await this.#registerDynamicScheduleService.call(endpoint, dynamicSchedule);
+
+        indexStats.dynamicSchedules++;
+      } catch (error) {
+        logger.error("Failed to register dynamic schedule", {
+          endpointId: endpoint.id,
+          dynamicSchedule,
+          error,
+        });
+      }
+    }
+
+    logger.debug("Endpoint indexing complete", {
+      endpointId: endpoint.id,
+      indexStats,
+      source: endpointIndex.source,
+      sourceData: endpointIndex.sourceData,
+      reason: endpointIndex.reason,
+    });
+
+    return await this.#prismaClient.endpointIndex.update({
+      where: {
+        id,
+      },
+      data: {
+        status: "SUCCESS",
+        stats: indexStats,
+        data: {
+          jobs,
+          sources,
+          dynamicTriggers,
+          dynamicSchedules,
+        },
+      },
+    });
+  }
+}
+
+async function updateEndpointIndexWithError(
+  prismaClient: PrismaClient,
+  id: string,
+  error: EndpointIndexError
+) {
+  return await prismaClient.endpointIndex.update({
+    where: {
+      id,
+    },
+    data: {
+      status: "FAILURE",
+      error,
+    },
+  });
+}

apps/webapp/app/services/endpoints/recurringEndpointIndex.server.ts

@@ -17,7 +17,9 @@ export class RecurringEndpointIndexService {
     const endpoints = await this.#prismaClient.endpoint.findMany({
       where: {
         environment: {
-          type: RuntimeEnvironmentType.PRODUCTION,
+          type: {
+            in: [RuntimeEnvironmentType.PRODUCTION, RuntimeEnvironmentType.STAGING],
+          },
         },
         indexings: {
           none: {
@@ -32,12 +34,18 @@ export class RecurringEndpointIndexService {
     logger.debug("Found endpoints that haven't been indexed in the last 10 minutes", {
       count: endpoints.length,
     });
-
     // Enqueue each endpoint for indexing
     for (const endpoint of endpoints) {
-      await workerQueue.enqueue("indexEndpoint", {
-        id: endpoint.id,
-        source: "INTERNAL",
+      const index = await this.#prismaClient.endpointIndex.create({
+        data: {
+          endpointId: endpoint.id,
+          status: "PENDING",
+          source: "INTERNAL",
+        },
+      });
+
+      await workerQueue.enqueue("performEndpointIndexing", {
+        id: index.id,
       });
     }
   }

apps/webapp/app/services/endpoints/validateCreateEndpoint.server.ts

@@ -66,12 +66,15 @@ export class ValidateCreateEndpointService {
           },
         });
 
-        // Kick off process to fetch the jobs for this endpoint
+        const index = await tx.endpointIndex.create({
+          data: { endpointId: endpoint.id, status: "PENDING", source: "INTERNAL" },
+        });
+
+        // Kick off process to fetch the jobs for this index
         await workerQueue.enqueue(
-          "indexEndpoint",
+          "performEndpointIndexing",
           {
-            id: endpoint.id,
-            source: "INTERNAL",
+            id: index.id,
           },
           {
             tx,

apps/webapp/app/services/jobs/registerJob.server.ts

@@ -4,14 +4,7 @@ import {
   SCHEDULED_EVENT,
   TriggerMetadata,
 } from "@trigger.dev/core";
-import type {
-  Endpoint,
-  Integration,
-  Job,
-  JobIntegration,
-  JobIntegrationPayload,
-  JobVersion,
-} from "@trigger.dev/database";
+import type { Endpoint, Integration, Job, JobIntegration, JobVersion } from "@trigger.dev/database";
 import { DEFAULT_MAX_CONCURRENT_RUNS } from "~/consts";
 import type { PrismaClient } from "~/db.server";
 import { prisma } from "~/db.server";
@@ -229,7 +222,7 @@ export class RegisterJobService {
           },
           update: {
             name: example.name,
-            icon: example.icon,
+            icon: example.icon ?? null,
             payload: example.payload,
           },
         });

apps/webapp/app/services/logger.server.ts

@@ -8,3 +8,10 @@ export const logger = new Logger(
   ["examples", "output", "connectionString", "payload"],
   sensitiveDataReplacer
 );
+
+export const workerLogger = new Logger(
+  "worker",
+  (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+  ["examples", "output", "connectionString"],
+  sensitiveDataReplacer
+);

apps/webapp/app/services/runs/createRun.server.ts

@@ -70,6 +70,7 @@ export class CreateRunService {
             ? eventRecord.externalAccountId
             : undefined,
           isTest: eventRecord.isTest,
+          internal: job.internal,
         },
       });
 

apps/webapp/app/services/schedules/nextScheduledEvent.server.ts

@@ -36,7 +36,7 @@ export class NextScheduledEventService {
 
       const scheduleTime = calculateNextScheduledEvent(
         schedule.data,
-        scheduleSource.lastEventTimestamp
+        scheduleSource.lastEventTimestamp ?? scheduleSource.createdAt
       );
 
       logger.debug("enqueuing scheduled event", {
@@ -67,6 +67,7 @@ export class NextScheduledEventService {
         },
         data: {
           workerJobId: workerJob.id,
+          nextEventTimestamp: scheduleTime,
         },
       });
 

apps/webapp/app/services/worker.server.ts

@@ -1,16 +1,18 @@
 import { DeliverEmailSchema } from "@/../../packages/emails/src";
-import { ScheduledPayloadSchema } from "@trigger.dev/core";
+import { ScheduledPayloadSchema, addMissingVersionField } from "@trigger.dev/core";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
 import { ZodWorker } from "~/platform/zodWorker.server";
 import { sendEmail } from "./email.server";
 import { IndexEndpointService } from "./endpoints/indexEndpoint.server";
+import { PerformEndpointIndexService } from "./endpoints/performEndpointIndexService";
 import { RecurringEndpointIndexService } from "./endpoints/recurringEndpointIndex.server";
 import { DeliverEventService } from "./events/deliverEvent.server";
 import { InvokeDispatcherService } from "./events/invokeDispatcher.server";
 import { integrationAuthRepository } from "./externalApis/integrationAuthRepository.server";
 import { IntegrationConnectionCreatedService } from "./externalApis/integrationConnectionCreated.server";
+import { logger } from "./logger.server";
 import { MissingConnectionCreatedService } from "./runs/missingConnectionCreated.server";
 import { PerformRunExecutionV1Service } from "./runs/performRunExecutionV1.server";
 import { PerformRunExecutionV2Service } from "./runs/performRunExecutionV2.server";
@@ -20,7 +22,6 @@ import { ActivateSourceService } from "./sources/activateSource.server";
 import { DeliverHttpSourceRequestService } from "./sources/deliverHttpSourceRequest.server";
 import { PerformTaskOperationService } from "./tasks/performTaskOperation.server";
 import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout";
-import { addMissingVersionField } from "@trigger.dev/core";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -29,6 +30,9 @@ const workerCatalog = {
     sourceData: z.any().optional(),
     reason: z.string().optional(),
   }),
+  performEndpointIndexing: z.object({
+    id: z.string(),
+  }),
   scheduleEmail: DeliverEmailSchema,
   startRun: z.object({ id: z.string() }),
   processCallbackTimeout: z.object({
@@ -128,6 +132,11 @@ function getWorkerQueue() {
   return new ZodWorker({
     name: "workerQueue",
     prisma,
+    cleanup: {
+      frequencyExpression: "13,27,43 * * * *",
+      ttl: 7 * 24 * 60 * 60 * 1000, // 7 days
+      maxCount: 1000,
+    },
     runnerOptions: {
       connectionString: env.DATABASE_URL,
       concurrency: env.WORKER_CONCURRENCY,
@@ -229,6 +238,7 @@ function getWorkerQueue() {
       deliverHttpSourceRequest: {
         priority: 1, // smaller number = higher priority
         maxAttempts: 14,
+        queueName: (payload) => `sources:${payload.id}`,
         handler: async (payload, job) => {
           const service = new DeliverHttpSourceRequestService();
 
@@ -255,7 +265,6 @@ function getWorkerQueue() {
       },
       performTaskOperation: {
         priority: 0, // smaller number = higher priority
-        queueName: (payload) => `tasks:${payload.id}`,
         maxAttempts: 3,
         handler: async (payload, job) => {
           const service = new PerformTaskOperationService();
@@ -264,7 +273,6 @@ function getWorkerQueue() {
         },
       },
       scheduleEmail: {
-        queueName: "internal-queue",
         priority: 100,
         maxAttempts: 3,
         handler: async (payload, job) => {
@@ -276,10 +284,17 @@ function getWorkerQueue() {
         maxAttempts: 7,
         handler: async (payload, job) => {
           const service = new IndexEndpointService();
-
           await service.call(payload.id, payload.source, payload.reason, payload.sourceData);
         },
       },
+      performEndpointIndexing: {
+        priority: 1, // smaller number = higher priority
+        maxAttempts: 7,
+        handler: async (payload, job) => {
+          const service = new PerformEndpointIndexService();
+          await service.call(payload.id);
+        },
+      },
       deliverEvent: {
         priority: 0, // smaller number = higher priority
         maxAttempts: 5,
@@ -291,7 +306,6 @@ function getWorkerQueue() {
       },
       refreshOAuthToken: {
         priority: 8, // smaller number = higher priority
-        queueName: "internal-queue",
         maxAttempts: 7,
         handler: async (payload, job) => {
           await integrationAuthRepository.refreshConnection({

apps/webapp/app/utils/icon.ts

@@ -0,0 +1,9 @@
+import { hasIcon } from "@trigger.dev/companyicons";
+import { iconNames as namedIcons } from "~/components/primitives/NamedIcon";
+
+export const isValidIcon = (icon?: string): boolean => {
+  if (!icon) {
+    return false;
+  }
+  return namedIcons.includes(icon) || hasIcon(icon);
+};

apps/webapp/app/utils/sse.ts

@@ -47,7 +47,7 @@ export function sse({ request, pingInterval = 1000, updateInterval = 348, run }:
             });
           }
         } else {
-          logger.debug("Uknown error sending SSE, aborting", {
+          logger.debug("Unknown error sending SSE, aborting", {
             error,
             args,
           });

apps/webapp/package.json

@@ -13,7 +13,7 @@
     "lint": "eslint --cache --cache-location ./node_modules/.cache/eslint .",
     "start": "cross-env NODE_ENV=production node --max-old-space-size=8192 ./build/server.js",
     "start:local": "cross-env node --max-old-space-size=8192 ./build/server.js",
-    "typecheck": "tsc --noEmit",
+    "typecheck": "tsc -p ./tsconfig.check.json",
     "db:seed": "node prisma/seed.js",
     "db:seed:local": "ts-node prisma/seed.ts",
     "generate:sourcemaps": "remix build --sourcemap",
@@ -106,13 +106,15 @@
     "simple-oauth2": "^5.0.0",
     "simplur": "^3.0.1",
     "slug": "^6.0.0",
+    "sonner": "^1.0.3",
     "tailwind-merge": "^1.12.0",
     "tailwind-scrollbar-hide": "^1.1.7",
     "tailwindcss-animate": "^1.0.5",
     "tiny-invariant": "^1.2.0",
     "ulid": "^2.3.0",
     "zod": "3.22.3",
-    "zod-error": "1.5.0"
+    "zod-error": "1.5.0",
+    "zod-validation-error": "^1.5.0"
   },
   "devDependencies": {
     "@remix-run/dev": "1.19.2-pre.0",

apps/webapp/tsconfig.check.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "paths": {
+      "~/*": ["./app/*"],
+      "@/*": ["./*"]
+    }
+  }
+}

docker/Dockerfile

@@ -32,7 +32,8 @@ ENV NODE_ENV production
 RUN pnpm install --prod --no-frozen-lockfile
 COPY --from=pruner --chown=node:node /triggerdotdev/packages/database/prisma/schema.prisma /triggerdotdev/packages/database/prisma/schema.prisma
 # RUN pnpm add @prisma/client@5.1.1 -w
-RUN pnpx prisma@4.16.0 generate --schema /triggerdotdev/packages/database/prisma/schema.prisma
+ENV NPM_CONFIG_IGNORE_WORKSPACE_ROOT_CHECK true
+RUN pnpx prisma@5.4.1 generate --schema /triggerdotdev/packages/database/prisma/schema.prisma
 
 ## Builder (builds the webapp)
 FROM base AS builder

docker/dev-compose.yml

@@ -35,6 +35,7 @@ services:
       DIRECT_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
       SESSION_SECRET: secret123
       MAGIC_LINK_SECRET: secret123
+      ENCRYPTION_KEY: secret123
       REMIX_APP_PORT: 3030
       PORT: 3030
     networks:

docker/docker-compose.yml

@@ -2,6 +2,7 @@ version: "3"
 
 volumes:
   database-data:
+  pgadmin-data:
 
 networks:
   app_network:
@@ -22,3 +23,21 @@ services:
       - app_network
     ports:
       - 5432:5432
+
+  pgadmin:
+    container_name: pgadmin
+    image: dpage/pgadmin4:7
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@example.com
+      PGADMIN_DEFAULT_PASSWORD: admin
+      PGADMIN_DISABLE_POSTFIX: "true"
+    volumes:
+      - pgadmin-data:/var/lib/pgadmin
+      - ./pgadmin/servers.json:/pgadmin4/servers.json
+    networks:
+      - app_network
+    ports:
+      - 5480:80
+    depends_on:
+      - database

docker/pgadmin/servers.json

@@ -0,0 +1,13 @@
+{
+  "Servers": {
+    "1": {
+      "Name": "Trigger.dev",
+      "Group": "Trigger.dev",
+      "Port": 5432,
+      "Username": "postgres",
+      "Host": "database",
+      "SSLMode": "prefer",
+      "MaintenanceDB": "postgres"
+    }
+  }
+}

docs/_snippets/manual-setup-sveltekit.mdx

@@ -1 +1,217 @@
-We're in the process of building support for the SvelteKit framework. You can follow along with progress or contribute via [this GitHub issue](https://github.com/triggerdotdev/trigger.dev/issues).
+## Installing Required Packages
+
+To begin, install the necessary packages in your Sveltekit project directory. You can choose one of the following package managers:
+
+<CodeGroup>
+
+```bash npm
+npm i @trigger.dev/sdk @trigger.dev/sveltekit
+```
+
+```bash pnpm
+pnpm install @trigger.dev/sdk @trigger.dev/sveltekit
+```
+
+```bash yarn
+yarn add @trigger.dev/sdk @trigger.dev/sveltekit
+```
+
+</CodeGroup>
+<br />
+
+<Note>Ensure that you execute this command within a SvelteKit project.</Note>
+## Obtaining the Development API Key
+
+To locate your development API key, login to the [Trigger.dev
+dashboard](https://cloud.trigger.dev) and select the Project you want to
+connect to. Then click on the Environments & API Keys tab in the left menu.
+You can copy your development API Key from the field at the top of this page.
+(Your development key will start with `tr_dev_`).
+
+## Adding Environment Variables
+
+Create a `.env` file at the root of your project and include your Trigger API key and URL like this:
+
+```bash
+TRIGGER_API_KEY=ENTER_YOUR_DEVELOPMENT_API_KEY_HERE
+TRIGGER_API_URL=https://api.trigger.dev # this is only necessary if you are self-hosting
+```
+
+Replace `ENTER_YOUR_DEVELOPMENT_API_KEY_HERE` with the actual API key obtained from the previous step.
+
+## Syncing Environment Variable types (TypeScript)
+
+You will have type errors for your environment variables unless you run this command:
+
+```sh
+npx svelte-kit sync
+```
+
+## Configuring the Trigger Client
+
+Create a file at `<root>/src/trigger.ts` or `<root>/trigger.ts` depending on whether you're using the `src` directory or not. `<root>` represents the root directory of your project.
+
+Next, add the following code to the file which creates and exports a new `TriggerClient`:
+
+```typescript src/trigger.(ts/js)
+// trigger.ts (for TypeScript) or trigger.js (for JavaScript)
+
+import { TriggerClient } from "@trigger.dev/sdk";
+import { TRIGGER_API_KEY, TRIGGER_API_URL } from "$env/static/private";
+
+export const client = new TriggerClient({
+  id: "my-app",
+  apiKey: TRIGGER_API_KEY,
+  apiUrl: TRIGGER_API_URL,
+});
+```
+
+Replace **"my-app"** with an appropriate identifier for your project.
+
+## Creating the API Route
+
+To establish an API route for interacting with Trigger.dev, follow these steps based on your project's file type and structure
+
+Create a new file named `+server.(ts/js)` within the `src/routes/api/trigger` directory, and add the following code:
+
+```typescript
+import { createSvelteRoute } from "@trigger.dev/sveltekit";
+import { client } from "../../../trigger";
+
+//import all jobs
+import "../../../jobs";
+
+// Create the Svelte route handler using the createSvelteRoute function
+const svelteRoute = createSvelteRoute(client);
+
+// Define your API route handler
+export const POST = svelteRoute.POST;
+```
+
+## Creating the Example Job
+
+1. Create a folder named `jobs` alongside your `src` directory
+2. Inside the `jobs` folder, add two files named `example.(ts/js)` and `index.(ts/js)`.
+
+<CodeGroup>
+
+```typescript src/jobs/example.(ts/js)
+import { eventTrigger } from "@trigger.dev/sdk";
+import { client } from "../trigger";
+
+// your first job
+client.defineJob({
+  id: "example-job",
+  name: "Example Job",
+  version: "0.0.1",
+  trigger: eventTrigger({
+    name: "example.event",
+  }),
+  run: async (payload, io, ctx) => {
+    await io.logger.info("Hello world!", { payload });
+
+    return {
+      message: "Hello world!",
+    };
+  },
+});
+```
+
+```typescript src/jobs/index.(ts/js)
+// export all your job files here
+export * from "./example";
+```
+
+</CodeGroup>
+
+## Additonal Job Definitions
+
+You can define more job definitions by creating additional files in the `jobs` folder and exporting them in the `src/jobs/index` file.
+
+For example, in `index.(ts/js)`, you can export other job files like this:
+
+```typescript
+// export all your job files here
+export * from "./example";
+export * from "./other-job-file";
+```
+
+## Adding Configuration to `package.json`
+
+Inside the `package.json` file, add the following configuration under the root object:
+
+```json
+"trigger.dev": {
+  "endpointId": "my-app"
+}
+```
+
+Your `package.json` file might look something like this:
+
+```json
+{
+  "name": "my-app",
+  "version": "1.0.0",
+  "dependencies": {
+    // ... other dependencies
+  },
+  "trigger.dev": {
+    "endpointId": "my-app"
+  }
+}
+```
+
+Replace **"my-app"** with the appropriate identifier you used during the step for creating the Trigger Client.
+
+## Running
+
+### Run your Sveltekit app
+
+Run your Sveltekit app locally. You need to use the `--host` flag to allow the Trigger.dev CLI to connect to your app.
+
+For example:
+
+<CodeGroup>
+
+```bash npm
+npm run dev -- --open --host
+```
+
+```bash pnpm
+pnpm run dev -- --open --host
+```
+
+```bash yarn
+yarn run dev -- --open --host
+```
+
+</CodeGroup>
+
+### Run the CLI 'dev' command
+
+In a **_separate terminal window or tab_** run:
+
+<CodeGroup>
+
+```bash npm
+npx @trigger.dev/cli@latest dev --port 5173
+```
+
+```bash pnpm
+pnpm dlx @trigger.dev/cli@latest dev --port 5173
+```
+
+```bash yarn
+yarn dlx @trigger.dev/cli@latest dev --port 5173
+```
+
+</CodeGroup>
+<br />
+<Note>
+  You can optionally pass the port if you're not running on 3000 by adding
+  `--port 5173` to the end
+</Note>
+<Note>
+  You can optionally pass the hostname if you're not running on localhost by adding
+  `--hostname <host>`. Example, in case your Sveltekit app is running on 0.0.0.0: `--hostname 0.0.0.0`.
+</Note>

docs/documentation/concepts/client-adaptors.mdx

@@ -24,11 +24,12 @@ Adaptors allows Clients to receive data from the Trigger API. They do this by cr
 
 Each platform has one or more adaptors, see the guides below:
 
-| Platform                                          | Adaptor              |
-| ------------------------------------------------- | -------------------- |
-| [Next.js](/documentation/guides/platforms/nextjs) | `createPagesRoute()` |
-| [Next.js](/documentation/guides/platforms/nextjs) | `createAppRoute()`   |
-| [NestJS](/documentation/guides/manual/nestjs)     | `TriggerDevModule`   |
-| [Astro](/documentation/guides/platforms/astro)    | `createAstroRoute()` |
-| [Remix](/documentation/guides/platforms/remix)    | `createRemixRoute()` |
-| Express                                           | Coming soon          |
+| Platform                                               | Adaptor               |
+| ------------------------------------------------------ | --------------------- |
+| [Next.js](/documentation/guides/platforms/nextjs)      | `createPagesRoute()`  |
+| [Next.js](/documentation/guides/platforms/nextjs)      | `createAppRoute()`    |
+| [NestJS](/documentation/guides/manual/nestjs)          | `TriggerDevModule`    |
+| [Astro](/documentation/guides/platforms/astro)         | `createAstroRoute()`  |
+| [Remix](/documentation/guides/platforms/remix)         | `createRemixRoute()`  |
+| [Sveltekit](/documentation/guides/platforms/sveltekit) | `createSvelteRoute()` |
+| Express                                                | Coming soon           |

docs/integrations/apis/github-tasks.mdx

@@ -1,54 +1,263 @@
 ---
-title: Tasks
+title: GitHub Tasks
+sidebarTitle: Tasks
+---
+
+Tasks are executed after the job is triggered and are the main building blocks of a job. You can string together as many tasks as you want.
+
 ---
 
 ## All tasks
 
-| Function Name                    | Description                                                 |
-| -------------------------------- | ----------------------------------------------------------- |
-| `createIssue`                    | Creates a new issue in a repository.                        |
-| `addIssueAssignees`              | Adds assignees to an existing issue.                        |
-| `addIssueLabels`                 | Adds labels to an existing issue.                           |
-| `createIssueComment`             | Creates a new comment on an existing issue.                 |
-| `getRepo`                        | Retrieves information about a repository.                   |
-| `createIssueCommentWithReaction` | Creates a new comment on an existing issue with a reaction. |
-| `addIssueCommentReaction`        | Adds a reaction to an existing issue comment.               |
-| `updateWebhook`                  | Updates an existing webhook.                                |
-| `createWebhook`                  | Creates a new webhook.                                      |
-| `listWebhooks`                   | Lists the webhooks for a repository.                        |
-| `updateOrgWebhook`               | Updates an existing webhook for an organization.            |
-| `createOrgWebhook`               | Creates a new webhook for an organization.                  |
-| `listOrgWebhooks`                | Lists the webhooks for an organization.                     |
+### `createIssue`
 
-## Usage
+Creates a new issue in a repository. [Official GitHub docs](https://docs.github.com/en/free-pro-team@latest/rest/issues/issues?apiVersion=2022-11-28#create-an-issue).
+
+```ts example.ts
+await io.github.createIssue("create issue", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  title: "<issue-title>", // the title of the issue
+  body: "<issue-description>", // the contents of the issue
+});
+```
+
+### `addIssueAssignees`
+
+Adds assignees to an existing issue. [Official GitHub docs](https://docs.github.com/en/free-pro-team@latest/rest/issues/assignees?apiVersion=2022-11-28#add-assignees-to-an-issue).
+
+```ts example.ts
+await io.github.addIssueAssignees("add assignee", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  issueNumber: <issue-number>, // the number of the issue
+  assignees: ["<assignee-name>"], // the name(s) of the assignee(s)
+});
+```
+
+### `addIssueLabels`
+
+Adds labels to an existing issue. [Official GitHub docs](https://docs.github.com/en/free-pro-team@latest/rest/issues/labels?apiVersion=2022-11-28#add-labels-to-an-issue).
+
+```ts example.ts
+await io.github.addIssueLabels("add label", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  issueNumber: <issue-number>, // the number of the issue
+  labels: ["<label-name>"], // the name(s) of the label(s)
+});
+```
+
+### `createIssueComment`
+
+Creates a new comment on an existing issue. [Official GitHub docs](https://docs.github.com/en/free-pro-team@latest/rest/issues/comments?apiVersion=2022-11-28#create-an-issue-comment).
+
+```ts example.ts
+await io.github.createIssueComment("create comment", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  issueNumber: <issue-number>, // the number of the issue
+  body: "<comment-text>", // the contents of the comment
+});
+```
+
+### `getRepo`
+
+Retrieves information about a repository. [Official GitHub docs](https://docs.github.com/en/free-pro-team@latest/rest/repos/repos?apiVersion=2022-11-28#get-a-repository).
+
+```ts example.ts
+const repoInfo = await io.github.getRepo({
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+});
+```
+
+### `createIssueCommentWithReaction`
+
+Creates a new comment on an existing issue with a reaction. [Official GitHub docs](https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#create-an-issue-comment).
+
+```ts example.ts
+await io.github.createIssueCommentWithReaction("create comment with reaction", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  issueNumber: <issue-number>, // the number of the issue
+  body: "<comment-text>", // the contents of the comment
+  content: "<reaction-type>", // the type of reaction
+});
+```
+
+### `addIssueCommentReaction`
+
+Adds a reaction to an existing issue comment. [Official GitHub docs](https://docs.github.com/en/free-pro-team@latest/rest/reactions/reactions?apiVersion=2022-11-28#create-reaction-for-an-issue-comment).
+
+```ts example.ts
+await io.github.addIssueCommentReaction("add reaction", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  commentId: <comment-id>, // the id of the specific comment
+  content: "<reaction-type>", // the type of reaction
+});
+```
+
+### `updateWebhook`
+
+Updates an existing webhook. [Official GitHub docs](https://docs.github.com/en/rest/webhooks/repos?apiVersion=2022-11-28#update-a-repository-webhook).
+
+```ts example.ts
+await io.github.updateWebhook("update webhook", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  webhookId: <webhook-id>, // the unique id of the webhook
+  config: {
+    url: "<webhook-url>", // the url to which payloads will be delivered
+    contentType: "json", // the media type used to serialize the payloads
+    secret: "<webhook-secret>", // If provided, the secret will be used as the key to generate the HMAC hex digest value for delivery signature headers.
+  },
+});
+```
+
+### `createWebhook`
+
+Creates a new webhook. [Official GitHub docs](https://docs.github.com/en/rest/webhooks/repos?apiVersion=2022-11-28#create-a-repository-webhook).
+
+```ts example.ts
+await io.github.createWebhook("create webhook", {
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+  config: {
+    url: "<webhook-url>", // the url to which payloads will be delivered
+    contentType: "json", // the media type used to serialize the payloads
+    secret: "<webhook-secret>", // If provided, the secret will be used as the key to generate the HMAC hex digest value for delivery signature headers.
+  },
+  events: ["<event-type>"], // the events for which the webhook will trigger
+});
+```
+
+### `listWebhooks`
+
+Lists the webhooks for a repository. [Official GitHub docs](https://docs.github.com/en/rest/webhooks/repos?apiVersion=2022-11-28#list-repository-webhooks).
+
+```ts example.ts
+const webhooks = await io.github.listWebhooks({
+  owner: "<owner-name>", // the name of the owner of the repository
+  repo: "<repo-name>", // the name of the repository
+});
+```
+
+### `updateOrgWebhook`
+
+Updates an existing webhook for an organization. [Official GitHub docs](https://docs.github.com/en/rest/orgs/webhooks?apiVersion=2022-11-28#update-an-organization-webhook).
+
+```ts
+await io.github.updateOrgWebhook("update org webhook", {
+  org: "<organization-name>", // the name of the organization
+  webhookId: <webhook-id>, // the unique id of the webhook
+  config: {
+    url: "<webhook-url>", // the url to which payloads will be delivered
+    contentType: "json", // the media type used to serialize the payloads
+    secret: "<webhook-secret>", // If provided, the secret will be used as the key to generate the HMAC hex digest value for delivery signature headers.
+  },
+});
+```
+
+### `createOrgWebhook`
+
+Creates a new webhook for an organization. [Official GitHub docs](https://docs.github.com/en/rest/orgs/webhooks?apiVersion=2022-11-28#create-an-organization-webhook).
+
+```ts example.ts
+await io.github.createOrgWebhook("create org webhook", {
+  org: "<organization-name>", // the name of the organization
+  config: {
+    url: "<webhook-url>", // the url to which payloads will be delivered
+    contentType: "json", // the media type used to serialize the payloads
+    secret: "<webhook-secret>", // If provided, the secret will be used as the key to generate the HMAC hex digest value for delivery signature headers.
+  },
+  events: ["<event-type>"], // the events for which the webhook will trigger
+});
+```
+
+### `listOrgWebhooks`
+
+Lists the webhooks for an organization. [Official GitHub docs](https://docs.github.com/en/rest/orgs/webhooks?apiVersion=2022-11-28#list-organization-webhooks).
+
+```ts example.ts
+const orgWebhooks = await io.github.listOrgWebhooks({
+  org: "<organization-name>", // the name of the organization
+  per-page: <number>, // the number of webhooks to return per page (max 100)
+  page: <number>, // Page number of the results to fetch.
+});
+```
+
+## Example usage
+
+In this example we'll create a task that adds an assignee and a label to an issue when it's opened.
 
 ```ts
 client.defineJob({
   id: "github-integration-on-issue-opened",
   name: "GitHub Integration - On Issue Opened",
-  version: "0.1.0",
+  version: "1.0.0",
   integrations: { github },
   trigger: github.triggers.repo({
     event: events.onIssueOpened,
-    owner: "triggerdotdev",
-    repo: "empty",
+    owner: "<your-org-name>",
+    repo: "<your-repo-name>",
   }),
   run: async (payload, io, ctx) => {
     await io.github.addIssueAssignees("add assignee", {
       owner: payload.repository.owner.login,
       repo: payload.repository.name,
       issueNumber: payload.issue.number,
-      assignees: ["matt-aitken"],
+      assignees: ["<assignee-name>"],
     });
 
     await io.github.addIssueLabels("add label", {
       owner: payload.repository.owner.login,
       repo: payload.repository.name,
       issueNumber: payload.issue.number,
-      labels: ["bug"],
+      labels: ["<label-name>"],
     });
 
     return { payload, ctx };
   },
 });
 ```
+
+## Using the underlying GitHub client
+
+You can access the [Octokit instance](https://github.com/octokit/octokit.js#octokit-api-client) by using the `runTask` method on the integration:
+
+```ts
+const github = new Github({
+  id: "github",
+});
+
+client.defineJob({
+  id: "github-example-1",
+  name: "GitHub Example 1",
+  version: "0.1.0",
+  trigger: eventTrigger({
+    name: "github.example",
+  }),
+  integrations: {
+    github,
+  },
+  run: async (payload, io, ctx) => {
+    const contributors = await io.github.runTask(
+      "get-contributors",
+      async (octokit, task) => {
+        const contributors = await octokit.rest.repos.listContributors({
+          owner: "<owner-name>",
+          repo: "<repo-name>",
+        });
+
+        return contributors;
+      },
+      //this is optional, it will appear on the Run page
+      { name: "List Contributors" }
+    );
+  },
+});
+```
+
+Make sure to pass the `idempotencyKey` to the underlying client to ensure that the API call is only executed once. This is only needed for mutating API calls.

docs/integrations/apis/github.mdx

@@ -1,11 +1,21 @@
 ---
-title: "GitHub: Introduction"
-sidebarTitle: "Introduction"
+title: GitHub overview & authentication
+sidebarTitle: Overview & authentication
 ---
 
-<Snippet file="integration-getting-started.mdx" />
+## Overview
 
-## Installation
+Our GitHub integration allows you to create triggers and tasks that interact with GitHub. For examples of some of the things you can do with it, check out our Jobs Showcase:
+
+<Card
+  title="Jobs Showcase - GitHub"
+  icon="rocket"
+  href="https://trigger.dev/showcase?tags=&integrations=github"
+>
+  Check out pre-built GitHub jobs in our showcase.
+</Card>
+
+## Installing the GitHub packages
 
 <CodeGroup>
 
@@ -25,25 +35,41 @@ yarn add @trigger.dev/github@latest
 
 ## Authentication
 
-GitHub supports Personal Access Tokens and OAuth.
+GitHub supports Personal Access Tokens and OAuth. You can use either of these to authenticate with GitHub.
 
-```ts
-import { Github } from "@trigger.dev/github";
+### Personal Access Token
+
+To create a personal access token on GitHub, login and [follow the instructions](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token). Information on the required scopes can be found [here](https://docs.github.com/en/developers/apps/scopes-for-oauth-apps).
+
+```ts my-job.ts
+import { GitHub } from "@trigger.dev/github";
 
 //create GitHub client using a token
-const github = new Github({
+const github = new GitHub({
   id: "github",
   token: process.env.GITHUB_TOKEN!,
 });
+...
+```
+
+### OAuth
+
+To use OAuth you can connect to GitHub via the Trigger.dev [web app](https://cloud.trigger.dev). Click 'Integrations' in the side panel of any project, and configure GitHub with the ID you want to use in your job and the required [scopes](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps).
+
+```ts my-job.ts
+import { GitHub } from "@trigger.dev/github";
 
 //create GitHub client using OAuth
-const github2 = new Github({
-  id: "github2",
+const github = new GitHub({
+  id: "github",
 });
+...
 ```
 
 ## Triggers and Tasks
 
+Once you have set up a GitHub client, you can use it to create triggers and tasks.
+
 <CardGroup cols={2}>
   <Card title="Triggers" icon="stars" href="/integrations/apis/github-triggers">
     Trigger Jobs when events happen in GitHub, such as a new commit or a new issue.
@@ -52,51 +78,3 @@ const github2 = new Github({
     Perform tasks such as creating a new issue or a new comment.
   </Card>
 </CardGroup>
-
-## Using the underlying client
-
-You can use the underlying client to do anything Octokit supports. In this example we create a project card when a new issue is opened..
-
-<Info>
-  View [the official GitHub docs](https://docs.github.com/en/rest) for everything that is
-  supported{" "}
-</Info>
-
-```ts
-import { Github, events } from "@trigger.dev/github";
-
-const github = new Github({
-  id: "github",
-  token: process.env.GITHUB_TOKEN!,
-});
-
-client.defineJob({
-  id: "alert-on-new-github-issues",
-  name: "Alert on new GitHub issues",
-  version: "0.1.1",
-  trigger: github.triggers.repo({
-    event: events.onIssueOpened,
-    owner: "triggerdotdev",
-    repo: "trigger.dev",
-  }),
-  integrations: {
-    github,
-  },
-  run: async (payload, io, ctx) => {
-    //io.github.runTask allows you to use the underlying SDK client
-    const { data } = await io.github.runTask(
-      "create-card",
-      async (client) => {
-        return client.rest.projects.createCard({
-          column_id: 123,
-          note: "test",
-        });
-      },
-      { name: "Create card" }
-    );
-
-    //log the url of the created card
-    await io.logger.info(data.url);
-  },
-});
-```

docs/integrations/apis/stripe.mdx

@@ -39,6 +39,8 @@ const stripe = new Stripe({
 
 The Stripe integration exposes a number of triggers that can be used on a job, powered by Stripe webhooks.
 
+We recommend testing Stripe payloads using [Stripe Shell](https://stripe.com/docs/stripe-cli?shell=true), Stripe's browser-based shell with the Stripe CLI pre-installed.
+
 ```ts
 client.defineJob({
   id: "stripe-price",

docs/integrations/apis/supabase/management.mdx

@@ -125,6 +125,7 @@ Now, you can use the `db` instance to add a trigger to run a job when a row is i
 client.defineJob({
   id: "supabase-trigger",
   name: "Supabase Trigger",
+  version: "1.0.0",
   trigger: db.onInserted({
     table: "todos",
   }),
@@ -140,6 +141,7 @@ You can add additional filters to the trigger by passing a `filter` object:
 client.defineJob({
   id: "supabase-trigger",
   name: "Supabase Trigger",
+  version: "1.0.0",
   trigger: db.onUpdated({
     table: "todos",
     // Only trigger if the todo is marked as completed
@@ -164,6 +166,7 @@ You can also listen for multiple different events using the `on` trigger:
 client.defineJob({
   id: "supabase-trigger",
   name: "Supabase Trigger",
+  version: "1.0.0",
   trigger: db.on({
     table: "todos",
     events: ["INSERT", "UPDATE"] // Trigger on both insert and update events
@@ -206,6 +209,7 @@ const db = supabase.db<Database>("https://<your project id>.supabase.co");
 client.defineJob({
   id: "supabase-trigger",
   name: "Supabase Trigger",
+  version: "1.0.0",
   trigger: db.onUpdated({
     table: "todos",
   }),

docs/integrations/introduction.mdx

@@ -33,6 +33,7 @@ Navigate the menu or select Integrations from the table below.
 
 | API                                       | Description                                                      | Webhooks | Tasks |
 | ----------------------------------------- | ---------------------------------------------------------------- | -------- | ----- |
+| [Airtable](/integrations/apis/airtable)   | Interact with the Airtable API                                   | 🕘       | ✅    |
 | [GitHub](/integrations/apis/github)       | Subscribe to webhooks and perform actions                        | ✅       | ✅    |
 | [Linear](/integrations/apis/linear)       | Streamline project and issue tracking                            | ✅       | ✅    |
 | [OpenAI](/integrations/apis/openai)       | Generate text and images. Including longer than 30s prompts      | N/A      | ✅    |
@@ -41,5 +42,6 @@ Navigate the menu or select Integrations from the table below.
 | [Resend](/integrations/apis/resend)       | Send emails using Resend                                         | 🕘       | ✅    |
 | [SendGrid](/integrations/apis/sendgrid)   | Send emails using SendGrid                                       | 🕘       | ✅    |
 | [Slack](/integrations/apis/slack)         | Send Slack messages                                              | 🕘       | ✅    |
+| [Stripe](/integrations/apis/stripe)       | Interact with the Stripe API                                     | ✅       | ✅    |
 | [Supabase](/integrations/apis/supabase)   | Interact with your projects and databases                        | ✅       | ✅    |
 | [Typeform](/integrations/apis/typeform)   | Interact with the Typeform API and get notified of new responses | ✅       | ✅    |

docs/mint.json

@@ -76,6 +76,7 @@
                 "documentation/quickstarts/express",
                 "documentation/quickstarts/remix",
                 "documentation/quickstarts/redwood",
+                "documentation/quickstarts/nestjs",
                 "documentation/quickstarts/astro",
                 "documentation/quickstarts/nuxt",
                 "documentation/quickstarts/sveltekit",
@@ -318,10 +319,7 @@
             "sdk/dynamictrigger/constructor",
             {
               "group": "Instance methods",
-              "pages": [
-                "sdk/dynamictrigger/register",
-                "sdk/dynamictrigger/unregister"
-              ]
+              "pages": ["sdk/dynamictrigger/register", "sdk/dynamictrigger/unregister"]
             }
           ]
         },
@@ -332,10 +330,7 @@
             "sdk/dynamicschedule/constructor",
             {
               "group": "Instance methods",
-              "pages": [
-                "sdk/dynamicschedule/register",
-                "sdk/dynamicschedule/unregister"
-              ]
+              "pages": ["sdk/dynamicschedule/register", "sdk/dynamicschedule/unregister"]
             }
           ]
         },
@@ -356,9 +351,7 @@
     },
     {
       "group": "Overview",
-      "pages": [
-        "examples/introduction"
-      ]
+      "pages": ["examples/introduction"]
     }
   ],
   "footerSocials": {
@@ -371,4 +364,4 @@
       "apiKey": "phc_hwYmedO564b3Ik8nhA4Csrb5SueY0EwFJWCbseGwWW"
     }
   }
-}
+}

docs/sdk/eventtrigger.mdx

@@ -45,6 +45,26 @@ You can have multiple Jobs that subscribe to the same event, they will all trigg
       ```
 
     </ResponseField>
+    <ResponseField name="examples" type="array">
+       Used to provide example payloads that are accepted by the job.
+
+       This will be available in the dashboard and can be used to trigger test runs.
+
+       <Expandable title="example object properties" defaultOpen>
+        <ResponseField name="id" type="string" required>
+          The example's ID.
+        </ResponseField>
+        <ResponseField name="name" type="string" required>
+          The name that's displayed in the dashboard.
+        </ResponseField>
+        <ResponseField name="payload" type="object" required>
+          The payload that's accepted by the job.
+        </ResponseField>
+        <ResponseField name="icon" type="string">
+          The icon to use for this example in the dashboard.
+        </ResponseField>
+      </Expandable>
+    </ResponseField>
 
   </Expandable>
 </ResponseField>
@@ -70,6 +90,19 @@ client.defineJob({
     filter: {
       tier: ["pro"],
     },
+    //(optional) example event object
+    examples: [
+      {
+        id: "issue.opened",
+        name: "Issue opened",
+        payload: {
+          userId: "1234",
+          tier: "free",
+        },
+        //optional
+        icon: "github",
+      },
+    ],
   }),
   run: async (payload, io, ctx) => {
     await io.logger.log("New pro user created", { userId: payload.userId });

docs/sdk/io/overview.mdx

@@ -64,5 +64,13 @@ If you want to send an event from outside a run (e.g. just from your backend) yo
 
 `io.registerTrigger()` allows you to register a [DynamicTrigger](/sdk/dynamictrigger) with the specified trigger data.
 
+### yield()
+
+`io.yield()` allows you to yield the current run and resume it immediately in a different function execution context. Requires a single argument that defines the yield key which works similar to task keys.
+
+### brb()
+
+`io.brb()` is is alias for `io.yield()`.
+
 {/* ### [unregisterTrigger()](/sdk/io/unregistertrigger) */}
 {/* `io.unregisterTrigger()` allows you to unregister a [DynamicTrigger](/sdk/dynamictrigger) that was previously registered with `io.registerTrigger()`. */}

docs/sdk/io/runtask.mdx

@@ -1,10 +1,10 @@
 ---
 title: "io.runTask()"
 sidebarTitle: "runTask()"
-description: "`io.runTask()` allows you to run a [Task](/documentation/concepts/tasks) from inside a Job run."
+description: "Creates and runs a Task inside a Run."
 ---
 
-A Task is a resumable unit of a Run that can be retried, resumed and is logged. [Integrations](/integrations) use Tasks internally to perform their actions.
+A [Task](/documentation/concepts/tasks) is a resumable unit of a Run that can be retried, resumed and is logged. [Integrations](/integrations) use Tasks internally to perform their actions.
 
 The wrappers at `io.integration.runTask()` expose the underlying Integration client as the first callback parameter (see examples on the right). They will have defaults set for options and `onError` handlers, but should otherwise be considered identical to raw `io.runTask()`.
 
@@ -128,6 +128,7 @@ The wrappers at `io.integration.runTask()` expose the underlying Integration cli
         The value of the property.
       </ResponseField>
     </Expandable>
+
   </ResponseField>
 
 </Expandable>
@@ -135,7 +136,7 @@ The wrappers at `io.integration.runTask()` expose the underlying Integration cli
 
 <ResponseField name="onError" type="function">
   An optional callback that will be called when the Task fails. You can perform
-  logic in here and optionally return a custom error object. Returning an object with `{ retryAt: Date, error?: Error }` will retry the Task at the specified Date. You can also just return a new `Error` object to throw a new error. Return nothing to rethrow the original error.
+  logic in here and optionally return a custom error object. Returning an object with `{ retryAt: Date, error?: Error }` will retry the Task at the specified Date. You can also just return a new `Error` object to throw a new error. Returning `null` or `undefined` will rethrow the original error. If you want to force retrying to be skipped, return `{ skipRetrying: true }`.
 
   <Expandable title="arguments">
     <ResponseField name="error" type="unknown">

integrations/resend/src/index.ts

@@ -25,6 +25,9 @@ function isRequestError(error: unknown): error is ErrorResponse {
   return typeof error === "object" && error !== null && "statusCode" in error;
 }
 
+// See https://resend.com/docs/api-reference/errors
+const skipRetryingErrors = [422, 401, 403, 404, 405, 422];
+
 function onError(error: unknown) {
   if (!isRequestError(error)) {
     if (error instanceof Error) {
@@ -34,6 +37,12 @@ function onError(error: unknown) {
     return new Error("Unknown error");
   }
 
+  if (skipRetryingErrors.includes(error.statusCode)) {
+    return {
+      skipRetrying: true,
+    };
+  }
+
   return new Error(error.message);
 }
 

packages/astro/package.json

@@ -33,7 +33,7 @@
     "typescript": "^4.8.4"
   },
   "engines": {
-    "node": ">=16.8.0"
+    "node": ">=18.0.0"
   },
   "license": "MIT",
   "publishConfig": {

packages/cli/package.json

@@ -56,11 +56,13 @@
     "test": "vitest"
   },
   "dependencies": {
+    "@trigger.dev/core": "workspace:*",
     "@types/degit": "^2.8.3",
     "boxen": "^7.1.1",
     "chalk": "^5.2.0",
     "chokidar": "^3.5.3",
     "commander": "^9.4.1",
+    "console-table-printer": "^2.11.2",
     "degit": "^2.8.4",
     "dotenv": "^16.3.1",
     "execa": "^7.0.0",
@@ -74,6 +76,7 @@
     "npm-check-updates": "^16.12.2",
     "openai": "^4.5.0",
     "ora": "^6.1.2",
+    "p-retry": "^6.1.0",
     "path-to-regexp": "^6.2.1",
     "posthog-node": "^3.1.1",
     "proxy-agent": "^6.3.0",
@@ -84,6 +87,6 @@
     "zod": "3.22.3"
   },
   "engines": {
-    "node": ">=16.8.0"
+    "node": ">=18.0.0"
   }
 }

packages/cli/src/cli/index.ts

@@ -93,10 +93,13 @@ program
 
 program
   .command("update")
-  .description("Updates all @trigger.dev/* packages to their latest compatible versions")
+  .description(
+    "Updates all @trigger.dev/* packages to their latest compatible versions or the specified version"
+  )
   .argument("[path]", "The path to the directory that contains the package.json file", ".")
-  .action(async (path) => {
-    await updateCommand(path);
+  .option("--to <version tag>", "The version to update to (ex: 2.1.4)", "latest")
+  .action(async (path, options) => {
+    await updateCommand(path, options);
   });
 
 program

packages/cli/src/commands/dev.ts

@@ -1,3 +1,4 @@
+import boxen from "boxen";
 import chalk from "chalk";
 import childProcess from "child_process";
 import chokidar from "chokidar";
@@ -5,10 +6,12 @@ import fs from "fs/promises";
 import ngrok from "ngrok";
 import { run as ncuRun } from "npm-check-updates";
 import ora, { Ora } from "ora";
+import pRetry, { AbortError } from "p-retry";
 import pathModule from "path";
 import util from "util";
 import { z } from "zod";
 import { Framework, getFramework } from "../frameworks";
+import { standardWatchFilePaths, standardWatchIgnoreRegex } from "../frameworks/watchConfig";
 import { telemetryClient } from "../telemetry/telemetry";
 import { getEnvFilename } from "../utils/env";
 import fetch from "../utils/fetchUseProxy";
@@ -17,8 +20,9 @@ import { getUserPackageManager } from "../utils/getUserPkgManager";
 import { logger } from "../utils/logger";
 import { resolvePath } from "../utils/parseNameAndPath";
 import { RequireKeys } from "../utils/requiredKeys";
+import { Throttle } from "../utils/throttle";
 import { TriggerApi } from "../utils/triggerApi";
-import { standardWatchIgnoreRegex, standardWatchFilePaths } from "../frameworks/watchConfig";
+import { wait } from "../utils/wait";
 
 const asyncExecFile = util.promisify(childProcess.execFile);
 
@@ -79,14 +83,14 @@ export async function devCommand(path: string, anyOptions: any) {
     telemetryClient.dev.failed("missing_api_key", resolvedOptions);
     return;
   }
-  const { apiUrl, envFile, apiKey } = apiDetails;
-  logger.success(`✔️ [trigger.dev] Found API Key in ${envFile} file`);
+  const { apiUrl, apiKey, apiKeySource } = apiDetails;
+  logger.success(`✔️ [trigger.dev] Found API Key in ${apiKeySource}`);
 
   //verify that the endpoint can be reached
   const verifiedEndpoint = await verifyEndpoint(resolvedOptions, endpointId, apiKey, framework);
   if (!verifiedEndpoint) {
     logger.error(
-      `✖ [trigger.dev] Failed to find a valid Trigger.dev endpoint. Make sure your app is running and try again.`
+      `✖ [trigger.dev] Your endpoint couldn't be verified. Make sure your app is running and try again. ${resolvedOptions.handlerPath}`
     );
     logger.info(`  [trigger.dev] You can use -H to specify a hostname, or -p to specify a port.`);
     telemetryClient.dev.failed("no_server_found", resolvedOptions);
@@ -107,83 +111,6 @@ export async function devCommand(path: string, anyOptions: any) {
   const endpointHandlerUrl = `${endpointUrl}${handlerPath}`;
   telemetryClient.dev.tunnelRunning(path, resolvedOptions);
 
-  const connectingSpinner = ora(`[trigger.dev] Registering endpoint ${endpointHandlerUrl}...`);
-
-  //refresh function
-  let hasConnected = false;
-  let attemptCount = 0;
-  const refresh = async () => {
-    connectingSpinner.start();
-
-    const refreshedEndpointId = await getEndpointIdFromPackageJson(resolvedPath, resolvedOptions);
-
-    // Read from env file to get the TRIGGER_API_KEY and TRIGGER_API_URL
-    const apiDetails = await getTriggerApiDetails(resolvedPath, envFile);
-
-    if (!apiDetails) {
-      connectingSpinner.fail(`[trigger.dev] Failed to connect: Missing API Key`);
-      logger.info(`Will attempt again on the next file change…`);
-      attemptCount = 0;
-      return;
-    }
-
-    const { apiKey, apiUrl } = apiDetails;
-    const apiClient = new TriggerApi(apiKey, apiUrl);
-
-    const authorizedKey = await apiClient.whoami(apiKey);
-    if (!authorizedKey) {
-      logger.error(
-        `✖ [trigger.dev] The API key you provided is not authorized. Try visiting your dashboard to get a new API key.`
-      );
-
-      telemetryClient.dev.failed("invalid_api_key", resolvedOptions);
-      return;
-    }
-
-    telemetryClient.identify(
-      authorizedKey.organization.id,
-      authorizedKey.project.id,
-      authorizedKey.userId
-    );
-
-    const result = await refreshEndpoint(
-      apiClient,
-      refreshedEndpointId ?? endpointId,
-      endpointHandlerUrl
-    );
-    if (result.success) {
-      attemptCount = 0;
-      connectingSpinner.succeed(
-        `[trigger.dev] 🔄 Refreshed ${refreshedEndpointId ?? endpointId} ${formattedDate.format(
-          new Date(result.data.updatedAt)
-        )}`
-      );
-
-      if (!hasConnected) {
-        hasConnected = true;
-        telemetryClient.dev.connected(path, resolvedOptions);
-      }
-    } else {
-      attemptCount++;
-
-      if (attemptCount === 10 || !result.retryable) {
-        connectingSpinner.fail(`Failed to connect: ${result.error}`);
-        logger.info(`Will attempt again on the next file change…`);
-        attemptCount = 0;
-
-        if (!hasConnected) {
-          telemetryClient.dev.failed("failed_to_connect", resolvedOptions);
-        }
-        return;
-      }
-
-      const delay = backoff(attemptCount);
-      // console.log(`Attempt: ${attemptCount}`, delay);
-      await wait(delay);
-      refresh();
-    }
-  };
-
   // Watch for changes to files and refresh endpoints
   const watchPaths = (framework?.watchFilePaths ?? standardWatchFilePaths).map(
     (path) => `${resolvedPath}/${path}`
@@ -195,12 +122,178 @@ export async function devCommand(path: string, anyOptions: any) {
     ignoreInitial: true,
   });
 
+  const connectingSpinner = ora(`[trigger.dev] Registering endpoint ${endpointHandlerUrl}...`);
+  let hasConnected = false;
+  const abortController = new AbortController();
+
+  const r = () => {
+    refresh({
+      endpointId,
+      spinner: connectingSpinner,
+      path: resolvedPath,
+      endpointHandlerUrl,
+      resolvedOptions,
+      hasConnected,
+      abortController,
+    });
+  };
+
+  const throttle = new Throttle(r, throttleTimeMs);
+
   watcher.on("all", (_event, _path) => {
-    throttle(refresh, throttleTimeMs);
+    throttle.call();
   });
 
   //Do initial refresh
-  throttle(refresh, throttleTimeMs);
+  throttle.call();
+}
+
+type RefreshOptions = {
+  spinner: Ora;
+  path: string;
+  endpointId: string;
+  endpointHandlerUrl: string;
+  resolvedOptions: ResolvedOptions;
+  hasConnected: boolean;
+  abortController: AbortController;
+};
+
+async function refresh(options: RefreshOptions) {
+  //stop any existing refreshes
+  options.abortController.abort();
+  options.abortController = new AbortController();
+
+  // Read from env file to get the TRIGGER_API_KEY and TRIGGER_API_URL
+  const apiDetails = await getTriggerApiDetails(options.path, options.resolvedOptions.envFile);
+  if (!apiDetails) {
+    options.spinner.fail("[trigger.dev] Failed to connect: Missing API Key");
+    return;
+  }
+
+  const { apiKey, apiUrl } = apiDetails;
+  const apiClient = new TriggerApi(apiKey, apiUrl);
+
+  try {
+    const index = await pRetry(() => startIndexing({ ...options, apiClient }), {
+      retries: 5,
+      signal: options.abortController.signal,
+      maxTimeout: 5000,
+    });
+    options.spinner.text = `[trigger.dev] Refreshing ${formattedDate.format(index.updatedAt)}`;
+
+    if (!options.hasConnected) {
+      options.hasConnected = true;
+      telemetryClient.dev.connected(options.path, options.resolvedOptions);
+    }
+
+    //this is for backwards-compatibility with older servers
+    if (index.id === undefined) {
+      options.spinner.succeed(`[trigger.dev] Refreshed ${formattedDate.format(index.updatedAt)}`);
+      return;
+    }
+
+    //wait 750ms before attempting to get the indexing result
+    await wait(750);
+
+    const indexResult = await pRetry(() => fetchIndexResult({ indexId: index.id, apiClient }), {
+      //this means we're polling, same distance between each attempt
+      factor: 1,
+      retries: 10,
+      signal: options.abortController.signal,
+    });
+
+    if (indexResult.status === "FAILURE") {
+      options.spinner.fail(
+        `[trigger.dev] Refreshing failed ${formattedDate.format(indexResult.updatedAt)}`
+      );
+      logger.error(
+        boxen(indexResult.error.message, {
+          padding: 1,
+          borderStyle: "double",
+        })
+      );
+      return;
+    }
+
+    options.spinner.succeed(
+      `[trigger.dev] Refreshed ${formattedDate.format(indexResult.updatedAt)}`
+    );
+  } catch (e) {
+    if (e instanceof AbortError) {
+      options.spinner.fail(e.message);
+      logger.info(`  [trigger.dev] Will attempt again on the next file change…`);
+      return;
+    }
+
+    let message: string = "";
+    if (e instanceof Error) {
+      message = e.message;
+    } else {
+      message = "Unknown error";
+    }
+
+    options.spinner.fail(message);
+    logger.info(`  [trigger.dev] Will attempt again on the next file change…`);
+
+    if (!options.hasConnected) {
+      telemetryClient.dev.failed("failed_to_connect", options.resolvedOptions);
+    }
+  }
+}
+
+async function startIndexing({
+  spinner,
+  path,
+  endpointId,
+  endpointHandlerUrl,
+  resolvedOptions,
+  apiClient,
+}: RefreshOptions & { apiClient: TriggerApi }) {
+  spinner.start();
+
+  const refreshedEndpointId = await getEndpointIdFromPackageJson(path, resolvedOptions);
+
+  const authorizedKey = await apiClient.whoami();
+  if (!authorizedKey) {
+    telemetryClient.dev.failed("invalid_api_key", resolvedOptions);
+    throw new AbortError(
+      "[trigger.dev] The API key you provided is not authorized. Try visiting your dashboard to get a new API key."
+    );
+  }
+
+  telemetryClient.identify(
+    authorizedKey.organization.id,
+    authorizedKey.project.id,
+    authorizedKey.userId
+  );
+
+  const result = await refreshEndpoint(
+    apiClient,
+    refreshedEndpointId ?? endpointId,
+    endpointHandlerUrl
+  );
+
+  if (!result.success) {
+    throw new Error(result.error);
+  }
+
+  return { id: result.data.endpointIndex?.id, updatedAt: new Date(result.data.updatedAt) };
+}
+
+async function fetchIndexResult({
+  indexId,
+  apiClient,
+}: {
+  indexId: string;
+  apiClient: TriggerApi;
+}) {
+  const result = await apiClient.getEndpointIndex(indexId);
+
+  if (result.status === "STARTED" || result.status === "PENDING") {
+    throw new Error("Indexing is still in progress");
+  }
+
+  return result;
 }
 
 async function resolveOptions(
@@ -209,7 +302,7 @@ async function resolveOptions(
   unresolvedOptions: DevCommandOptions
 ): Promise<ResolvedOptions> {
   if (!framework) {
-    logger.info("Failed to detect framework, using default values");
+    logger.info("  [trigger.dev] Failed to detect framework, using default values");
     return {
       port: unresolvedOptions.port ?? 3000,
       hostname: unresolvedOptions.hostname ?? "localhost",
@@ -431,25 +524,3 @@ async function refreshEndpoint(apiClient: TriggerApi, endpointId: string, endpoi
     }
   }
 }
-
-//wait function
-async function wait(ms: number) {
-  return new Promise((resolve) => {
-    setTimeout(resolve, ms);
-  });
-}
-
-//throttle function
-let throttleTimeout: NodeJS.Timeout | null = null;
-function throttle(fn: () => any, delay: number) {
-  if (throttleTimeout) {
-    clearTimeout(throttleTimeout);
-  }
-  throttleTimeout = setTimeout(fn, delay);
-}
-
-const maximum_backoff = 30;
-const initial_backoff = 0.2;
-function backoff(attempt: number) {
-  return Math.min((2 ^ attempt) * initial_backoff, maximum_backoff) * 1000;
-}

packages/cli/src/commands/init.ts

@@ -81,7 +81,7 @@ export const initCommand = async (options: InitCommandOptions) => {
   }
 
   const apiClient = new TriggerApi(apiKey, optionsAfterPrompts.apiUrl);
-  const authorizedKey = await apiClient.whoami(apiKey);
+  const authorizedKey = await apiClient.whoami();
 
   if (!authorizedKey) {
     logger.error(

packages/cli/src/commands/sendEvent.ts

@@ -36,7 +36,7 @@ export async function sendEventCommand(path: string, anyOptions: any) {
     return;
   }
 
-  const { apiUrl, envFile, apiKey } = apiDetails;
+  const { apiUrl, apiKey } = apiDetails;
 
   const parsedPayload = safeJSONParse(options.payload);
 

packages/cli/src/commands/update.ts

@@ -4,8 +4,24 @@ import { run, RunOptions } from "npm-check-updates";
 import { installDependencies } from "../utils/installDependencies.js";
 import { readJSONFileSync, writeJSONFile } from "../utils/fileSystem.js";
 import { logger } from "../utils/logger.js";
+import { z } from "zod";
+
+export const UpdateCommandOptionsSchema = z.object({
+  to: z.string().optional(),
+});
+
+export type UpdateCommandOptions = z.infer<typeof UpdateCommandOptionsSchema>;
+
+type NcuRunOptionTarget = "latest" | `@${string}`;
+
+export async function updateCommand(projectPath: string, anyOptions: any) {
+  const parseRes = UpdateCommandOptionsSchema.safeParse(anyOptions);
+  if (!parseRes.success) {
+    logger.error(parseRes.error.message);
+    return;
+  }
+  const options = parseRes.data;
 
-export async function updateCommand(projectPath: string) {
   const triggerDevPackage = "@trigger.dev";
   const packageJSONPath = path.join(projectPath, "package.json");
   const packageData = readJSONFileSync(packageJSONPath);
@@ -27,12 +43,14 @@ export async function updateCommand(projectPath: string) {
     };
   });
 
+  const targetVersion = getTargetVersion(options.to);
+
   // Use npm-check-updates to get updated dependency versions
   const ncuOptions: RunOptions = {
     packageData,
     upgrade: true,
     jsonUpgraded: true,
-    target: "latest",
+    target: targetVersion,
   };
 
   // Can either give a json like package.json or just with deps and their new versions
@@ -69,6 +87,39 @@ export async function updateCommand(projectPath: string) {
     return;
   }
 
+  let applyUpdates = targetVersion !== "latest";
+
+  if (targetVersion === "latest") {
+    applyUpdates = await hasUserConfirmed(packagesToUpdate, packageMaps, updatedDependencies);
+  }
+
+  if (applyUpdates) {
+    const newPackageJSON = packageData;
+    packagesToUpdate.forEach((packageName) => {
+      const tmp = packageMaps[packageName];
+      if (tmp) {
+        newPackageJSON[tmp.type][packageName] = updatedDependencies[packageName];
+      }
+    });
+    await writeJSONFile(packageJSONPath, newPackageJSON);
+    await installDependencies(projectPath);
+  }
+}
+
+// expects a version number, or latest.
+// if version number is specified, prepend it with '@' for ncu.
+function getTargetVersion(toVersion?: string): NcuRunOptionTarget {
+  if (!toVersion) {
+    return "latest";
+  }
+  return toVersion === "latest" ? "latest" : `@${toVersion}`;
+}
+
+async function hasUserConfirmed(
+  packagesToUpdate: string[],
+  packageMaps: { [x: string]: { type: string; version: string } },
+  updatedDependencies: { [x: string]: any }
+): Promise<boolean> {
   // Inform the user of the dependencies that can be updated
   console.log("\nNewer versions found for the following packages:");
   console.table(
@@ -86,15 +137,5 @@ export async function updateCommand(projectPath: string) {
     message: "Do you want to update these packages in package.json and re-install dependencies?",
   });
 
-  if (confirm) {
-    const newPackageJSON = packageData;
-    packagesToUpdate.forEach((packageName) => {
-      const tmp = packageMaps[packageName];
-      if (tmp) {
-        newPackageJSON[tmp.type][packageName] = updatedDependencies[packageName];
-      }
-    });
-    await writeJSONFile(packageJSONPath, newPackageJSON);
-    await installDependencies(projectPath);
-  }
+  return confirm;
 }

packages/cli/src/commands/whoami.ts

@@ -42,7 +42,7 @@ export async function whoamiCommand(path: string, anyOptions: any) {
   }
 
   const triggerAPI = new TriggerApi(apiDetails.apiKey, apiDetails.apiUrl);
-  const userData = await triggerAPI.whoami(apiDetails.apiKey);
+  const userData = await triggerAPI.whoami();
 
   loadingSpinner.stop();
 

packages/cli/src/utils/getTriggerApiDetails.ts

@@ -1,66 +1,23 @@
-import pathModule from "path";
-import { pathExists, readFile } from "./fileSystem";
 import { logger } from "./logger";
-import dotenv from "dotenv";
 import { CLOUD_API_URL } from "../consts";
 import { checkApiKeyIsDevServer } from "./getApiKeyType";
-
-export async function readEnvFilesWithBackups(
-  path: string,
-  envFile: string,
-  backups: string[]
-): Promise<{ content: string; fileName: string } | undefined> {
-  const envFilePath = pathModule.join(path, envFile);
-  const envFileExists = await pathExists(envFilePath);
-
-  if (envFileExists) {
-    const content = await readFile(envFilePath);
-
-    return { content, fileName: envFile };
-  }
-
-  for (const backup of backups) {
-    const backupPath = pathModule.join(path, backup);
-    const backupExists = await pathExists(backupPath);
-
-    if (backupExists) {
-      const content = await readFile(backupPath);
-
-      return { content, fileName: backup };
-    }
-  }
-
-  return;
-}
+import { readEnvVariables } from "./readEnvVariables";
 
 export async function getTriggerApiDetails(path: string, envFile: string) {
-  const resolvedEnvFile = await readEnvFilesWithBackups(path, envFile, [
-    ".env",
-    ".env.local",
-    ".env.development.local",
-  ]);
+  const envVarsToRead = ["TRIGGER_API_KEY", "TRIGGER_API_URL"];
+  const resolvedEnvVars = await readEnvVariables(path, envFile, envVarsToRead);
 
-  if (!resolvedEnvFile) {
-    logger.error(`You must add TRIGGER_API_KEY to your ${envFile} file.`);
-    return;
-  }
-
-  const parsedEnvFile = dotenv.parse(resolvedEnvFile.content);
-
-  if (!parsedEnvFile) {
-    logger.error(`You must add TRIGGER_API_KEY to your ${envFile} file.`);
-    return;
-  }
-
-  const apiKey = parsedEnvFile.TRIGGER_API_KEY;
-  const apiUrl = parsedEnvFile.TRIGGER_API_URL;
+  const apiKey = resolvedEnvVars.TRIGGER_API_KEY;
+  const apiUrl = resolvedEnvVars.TRIGGER_API_URL;
 
   if (!apiKey) {
-    logger.error(`You must add TRIGGER_API_KEY to your ${envFile} file.`);
+    logger.error(
+      `You must add TRIGGER_API_KEY to your ${envFile} file or set as runtime environment variable.`
+    );
     return;
   }
 
-  const result = checkApiKeyIsDevServer(apiKey);
+  const result = checkApiKeyIsDevServer(apiKey.value);
 
   if (!result.success) {
     if (result.type) {
@@ -75,5 +32,10 @@ export async function getTriggerApiDetails(path: string, envFile: string) {
     return;
   }
 
-  return { apiKey, apiUrl: apiUrl ?? CLOUD_API_URL, envFile: resolvedEnvFile.fileName };
+  return {
+    apiKey: apiKey.value,
+    apiUrl: apiUrl?.value ?? CLOUD_API_URL,
+    apiKeySource:
+      apiKey.source.type === "runtime" ? "process runtime" : `${apiKey.source.name} file`,
+  };
 }

packages/cli/src/utils/logger.ts

@@ -13,4 +13,7 @@ export const logger = {
   success(...args: unknown[]) {
     console.log(chalk.green(...args));
   },
+  table(rows: any) {
+    console.table(rows);
+  },
 };

packages/cli/src/utils/readEnvVariables.ts

@@ -0,0 +1,102 @@
+import pathModule from "path";
+import { pathExists, readFile } from "./fileSystem";
+import dotenv from "dotenv";
+
+const ENV_FILES_FALLBACK = [".env", ".env.local", ".env.development.local"];
+
+export type EnvVarSourceRuntime = {
+  type: "runtime";
+};
+
+export type EnvVarSourceFile = {
+  type: "file";
+  name: string;
+};
+
+export type EnvVarSource = EnvVarSourceRuntime | EnvVarSourceFile;
+
+export type EnvironmentVariable = {
+  value: string;
+  source: EnvVarSource;
+};
+
+export type EnvironmentVariables = {
+  [name: string]: EnvironmentVariable | undefined;
+};
+
+// Reads `varsToRead` from `process.env` and `envFile` (with fallbacks).
+// `process.env` takes precedence over the `envFile`.
+export async function readEnvVariables(
+  path: string,
+  envFile: string,
+  varsToRead: string[]
+): Promise<EnvironmentVariables> {
+  const resolvedEnvFile = await readEnvFilesWithBackups(path, envFile);
+  const parsedEnvFile = resolvedEnvFile
+    ? { output: dotenv.parse(resolvedEnvFile.content), filename: resolvedEnvFile.fileName }
+    : {};
+
+  return Object.fromEntries(
+    varsToRead.map((envVar) => [
+      envVar,
+      readFromRuntime(envVar) ?? readFromFile(envVar, parsedEnvFile),
+    ])
+  );
+}
+
+async function readEnvFilesWithBackups(
+  path: string,
+  envFile: string
+): Promise<{ content: string; fileName: string } | undefined> {
+  const envFilePath = pathModule.join(path, envFile);
+  const envFileExists = await pathExists(envFilePath);
+
+  if (envFileExists) {
+    const content = await readFile(envFilePath);
+
+    return { content, fileName: envFile };
+  }
+
+  for (const fallBack of ENV_FILES_FALLBACK) {
+    const fallbackPath = pathModule.join(path, fallBack);
+    const fallbackExists = await pathExists(fallbackPath);
+
+    if (fallbackExists) {
+      const content = await readFile(fallbackPath);
+
+      return { content, fileName: fallBack };
+    }
+  }
+
+  return;
+}
+
+function readFromRuntime(envVar: string): EnvironmentVariable | undefined {
+  const val = process.env[envVar];
+  if (!val) {
+    return;
+  }
+  return {
+    value: val,
+    source: {
+      type: "runtime",
+    } as EnvVarSourceRuntime,
+  };
+}
+
+function readFromFile(
+  envVar: string,
+  parsedEnvFile: { output?: dotenv.DotenvParseOutput; filename?: string }
+): EnvironmentVariable | undefined {
+  const val = parsedEnvFile.output ? parsedEnvFile.output[envVar] : undefined;
+  if (!val) {
+    return;
+  }
+  return {
+    value: val,
+    source: {
+      type: "file",
+      name: parsedEnvFile.filename,
+    } as EnvVarSourceFile,
+  };
+}

packages/cli/src/utils/throttle.ts

@@ -0,0 +1,18 @@
+export class Throttle {
+  throttleTimeout: NodeJS.Timeout | null = null;
+
+  constructor(
+    private readonly fn: () => any,
+    private readonly delay: number
+  ) {
+    this.fn = fn;
+    this.delay = delay;
+  }
+
+  call() {
+    if (this.throttleTimeout) {
+      clearTimeout(this.throttleTimeout);
+    }
+    this.throttleTimeout = setTimeout(this.fn, this.delay);
+  }
+}

packages/cli/src/utils/triggerApi.ts

@@ -1,5 +1,7 @@
 import fetch from "./fetchUseProxy";
 import { z } from "zod";
+import core from "@trigger.dev/core";
+const { GetEndpointIndexResponseSchema } = core;
 
 export type CreateEndpointOptions = {
   id: string;
@@ -16,6 +18,9 @@ export type EndpointData = {
   createdAt: string;
   updatedAt: string;
   indexingHookIdentifier: string;
+  endpointIndex: {
+    id: string;
+  };
 };
 
 export type EndpointResponse =
@@ -59,12 +64,12 @@ export class TriggerApi {
     private baseUrl: string
   ) {}
 
-  async whoami(apiKey: string): Promise<WhoamiResponse | undefined> {
+  async whoami(): Promise<WhoamiResponse | undefined> {
     const response = await fetch(`${this.baseUrl}/api/v1/whoami`, {
       method: "GET",
       headers: {
         Accept: "application/json",
-        Authorization: `Bearer ${apiKey}`,
+        Authorization: `Bearer ${this.apiKey}`,
       },
     });
 
@@ -155,6 +160,33 @@ export class TriggerApi {
       data: data as any as EndpointData,
     };
   }
+
+  async getEndpointIndex(indexId: string) {
+    const response = await fetch(`${this.baseUrl}/api/v1/endpointindex/${indexId}`, {
+      method: "GET",
+      headers: {
+        Accept: "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+    });
+
+    if (response.ok) {
+      const body = await response.json();
+      const parsed = GetEndpointIndexResponseSchema.safeParse(body);
+
+      if (parsed.success) {
+        return parsed.data;
+      }
+    }
+
+    return {
+      status: "FAILURE" as const,
+      error: {
+        message: `Bad response from Trigger.dev (${response.status})`,
+      },
+      updatedAt: new Date(),
+    };
+  }
 }
 
 function safeJsonParse(raw: string | null | undefined): unknown {

packages/cli/src/utils/wait.ts

@@ -0,0 +1,5 @@
+export async function wait(ms: number) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}

packages/cli/tsconfig.json

@@ -9,7 +9,6 @@
     "resolveJsonModule": true,
     "allowJs": true,
     "checkJs": true,
-
     /* EMIT RULES */
     "outDir": "./dist",
     "noEmit": true, // TSUP takes care of emitting js for us, in a MUCH faster way

packages/core/package.json

@@ -42,6 +42,6 @@
     "typescript": "^4.9.4"
   },
   "engines": {
-    "node": ">=16.8.0"
+    "node": ">=18.0.0"
   }
 }

packages/core/src/schemas/api.ts

@@ -296,6 +296,53 @@ export const IndexEndpointResponseSchema = z.object({
 
 export type IndexEndpointResponse = z.infer<typeof IndexEndpointResponseSchema>;
 
+export const EndpointIndexErrorSchema = z.object({
+  message: z.string(),
+  raw: z.any().optional(),
+});
+
+export type EndpointIndexError = z.infer<typeof EndpointIndexErrorSchema>;
+
+const IndexEndpointStatsSchema = z.object({
+  jobs: z.number(),
+  sources: z.number(),
+  dynamicTriggers: z.number(),
+  dynamicSchedules: z.number(),
+  disabledJobs: z.number().default(0),
+});
+
+export type IndexEndpointStats = z.infer<typeof IndexEndpointStatsSchema>;
+
+export function parseEndpointIndexStats(stats: unknown): IndexEndpointStats | undefined {
+  if (stats === null || stats === undefined) {
+    return;
+  }
+  return IndexEndpointStatsSchema.parse(stats);
+}
+
+export const GetEndpointIndexResponseSchema = z.discriminatedUnion("status", [
+  z.object({
+    status: z.literal("PENDING"),
+    updatedAt: z.coerce.date(),
+  }),
+  z.object({
+    status: z.literal("STARTED"),
+    updatedAt: z.coerce.date(),
+  }),
+  z.object({
+    status: z.literal("SUCCESS"),
+    stats: IndexEndpointStatsSchema,
+    updatedAt: z.coerce.date(),
+  }),
+  z.object({
+    status: z.literal("FAILURE"),
+    error: EndpointIndexErrorSchema,
+    updatedAt: z.coerce.date(),
+  }),
+]);
+
+export type GetEndpointIndexResponse = z.infer<typeof GetEndpointIndexResponseSchema>;
+
 export const EndpointHeadersSchema = z.object({
   "trigger-version": z.string().optional(),
 });
@@ -664,6 +711,7 @@ export const RunTaskBodyInputSchema = RunTaskOptionsSchema.extend({
 export type RunTaskBodyInput = z.infer<typeof RunTaskBodyInputSchema>;
 
 export const RunTaskBodyOutputSchema = RunTaskBodyInputSchema.extend({
+  properties: z.array(DisplayPropertySchema.partial()).optional(),
   params: DeserializedJsonSchema.optional().nullable(),
   callback: z
     .object({

packages/database/package.json

@@ -5,17 +5,18 @@
   "main": "./src/index.ts",
   "types": "./src/index.ts",
   "dependencies": {
-    "@prisma/client": "4.16.0",
+    "@prisma/client": "5.4.1",
     "typescript": "^4.8.4"
   },
   "devDependencies": {
-    "prisma": "4.16.0"
+    "prisma": "5.4.1"
   },
   "scripts": {
     "generate": "prisma generate",
     "db:migrate:dev": "prisma migrate dev",
+    "db:migrate:dev:create": "prisma migrate dev --create-only",
     "db:migrate:deploy": "prisma migrate deploy",
     "db:studio": "prisma studio",
     "typecheck": "tsc --noEmit"
   }
-}
+}

packages/database/prisma/migrations/20231005064823_add_job_run_internal/migration.sql

@@ -0,0 +1,10 @@
+-- AlterTable
+ALTER TABLE "JobRun" ADD COLUMN     "internal" BOOLEAN NOT NULL DEFAULT false;
+
+/*
+  Backfill JobRun internal flag
+*/
+UPDATE "JobRun"
+SET "internal" = "Job"."internal"
+FROM "Job"
+WHERE "JobRun"."jobId" = "Job"."id" AND "JobRun"."internal" = TRUE;

packages/database/prisma/migrations/20231010115840_endpoint_index_status_added/migration.sql

@@ -0,0 +1,11 @@
+-- CreateEnum
+CREATE TYPE "EndpointIndexStatus" AS ENUM ('PENDING', 'STARTED', 'SUCCESS', 'FAILURE');
+
+-- AlterTable
+ALTER TABLE "EndpointIndex"
+ADD COLUMN "status" "EndpointIndexStatus" NOT NULL DEFAULT 'PENDING';
+
+-- Update all existing rows to be SUCCESS. This isn't correct because some of them have failed, but we don't want them to be PENDING.
+UPDATE "EndpointIndex"
+SET
+  "status" = 'SUCCESS';

packages/database/prisma/migrations/20231010120458_endpoint_index_data_and_stats_are_now_optional/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "EndpointIndex" ALTER COLUMN "data" DROP NOT NULL,
+ALTER COLUMN "stats" DROP NOT NULL;