Fix pnpm lock file

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/proxy/.dev.vars.example

@@ -0,0 +1,7 @@
+REWRITE_HOSTNAME=
+AWS_SQS_ACCESS_KEY_ID=
+AWS_SQS_SECRET_ACCESS_KEY=
+AWS_SQS_QUEUE_URL=
+AWS_SQS_REGION=
+#optional
+#REWRITE_PORT=

apps/proxy/.editorconfig

@@ -0,0 +1,13 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = tab
+tab_width = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.yml]
+indent_style = space

apps/proxy/.gitignore

@@ -0,0 +1,172 @@
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+\*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+\*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+\*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+\*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.cache
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+.cache/
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+.cache
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.\*
+
+# wrangler project
+
+.dev.vars
+.wrangler/

apps/proxy/.prettierrc

@@ -0,0 +1,11 @@
+{
+	"semi": true,
+	"singleQuote": false,
+	"jsxSingleQuote": false,
+	"trailingComma": "es5",
+	"bracketSpacing": true,
+	"bracketSameLine": false,
+	"printWidth": 100,
+	"tabWidth": 2,
+	"useTabs": false
+}

apps/proxy/CHANGELOG.md

@@ -0,0 +1,8 @@
+# proxy
+
+## 0.0.1
+
+### Patch Changes
+
+- Updated dependencies [756024da]
+  - @trigger.dev/core@2.2.7

apps/proxy/README.md

@@ -0,0 +1,68 @@
+# Trigger.dev proxy
+
+This is an optional module that can be used to proxy and queue requests to the Trigger.dev API.
+
+## Why?
+
+The Trigger.dev API is designed to be fast and reliable. However, if you have a lot of traffic, you may want to use this proxy to queue requests to the API. It intercepts some requests to the API and adds them to an AWS SQS queue, then the webapp can be setup to process the queue.
+
+## Current features
+
+- Intercepts `sendEvent` requests and adds them to an AWS SQS queue. The webapp then reads from the queue and creates the events.
+
+## Setup
+
+### Create an AWS SQS queue
+
+In AWS you should create a new AWS SQS queue with appropriate security settings. You will need the queue URL for the next step.
+
+### Environment variables
+
+#### Cloudflare secrets
+
+Locally you should copy the `.dev.var.example` file to `.dev.var` and fill in the values.
+
+When deploying you should use `wrangler` (the Cloudflare CLI tool) to set secrets. Make sure you set the correct --env ("staging" or "prod")
+
+```bash
+wrangler secret put REWRITE_HOSTNAME --env staging
+wrangler secret put AWS_SQS_ACCESS_KEY_ID --env staging
+wrangler secret put AWS_SQS_SECRET_ACCESS_KEY --env staging
+wrangler secret put AWS_SQS_QUEUE_URL --env staging
+wrangler secret put AWS_SQS_REGION --env staging
+```
+
+You need to set your API CNAME entry to be proxied by Cloudflare. You can do this in the Cloudflare dashboard.
+
+#### Webapp
+
+These env vars also need setting in the webapp.
+
+```bash
+AWS_SQS_REGION
+AWS_SQS_ACCESS_KEY_ID
+AWS_SQS_SECRET_ACCESS_KEY
+AWS_SQS_QUEUE_URL
+AWS_SQS_BATCH_SIZE
+```
+
+## Deployment
+
+Staging:
+
+```bash
+npx wrangler@latest deploy --route "<your-api-subdomain>/*" --env staging
+```
+
+Prod:
+
+```bash
+npx wrangler@latest deploy --route "<your-api-subdomain>/*" --env prod
+```
+
+## Development
+
+Set the environment variables as described above.
+
+1. `pnpm install`
+2. `pnpm run dev --filter proxy`

apps/proxy/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "proxy",
+  "version": "0.0.1",
+  "private": true,
+  "scripts": {
+    "deploy": "wrangler deploy",
+    "dev": "wrangler dev"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20230419.0",
+    "typescript": "^5.0.4",
+    "wrangler": "^3.0.0"
+  },
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.445.0",
+    "@trigger.dev/core": "workspace:*",
+    "ulidx": "^2.2.1",
+    "zod": "3.22.3",
+    "zod-error": "1.5.0"
+  }
+}

apps/proxy/src/apikey.ts

@@ -0,0 +1,20 @@
+import { z } from "zod";
+
+const AuthorizationHeaderSchema = z.string().regex(/^Bearer .+$/);
+
+export function getApiKeyFromRequest(request: Request) {
+  const rawAuthorization = request.headers.get("Authorization");
+
+  const authorization = AuthorizationHeaderSchema.safeParse(rawAuthorization);
+  if (!authorization.success) {
+    return;
+  }
+
+  const apiKey = authorization.data.replace(/^Bearer /, "");
+  const type = isPrivateApiKey(apiKey) ? ("PRIVATE" as const) : ("PUBLIC" as const);
+  return { apiKey, type };
+}
+
+function isPrivateApiKey(key: string) {
+  return key.startsWith("tr_");
+}

apps/proxy/src/events/queueEvent.ts

@@ -0,0 +1,87 @@
+import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";
+import { ApiEventLog, SendEventBodySchema } from "@trigger.dev/core";
+import { generateErrorMessage } from "zod-error";
+import { Env } from "..";
+import { getApiKeyFromRequest } from "../apikey";
+import { json } from "../json";
+import { calculateDeliverAt } from "./utils";
+
+/** Adds the event to an AWS SQS queue, so it can be consumed from the main Trigger.dev API */
+export async function queueEvent(request: Request, env: Env): Promise<Response> {
+  //check there's a private API key
+  const apiKeyResult = getApiKeyFromRequest(request);
+  if (!apiKeyResult || apiKeyResult.type !== "PRIVATE") {
+    return json(
+      { error: "Invalid or Missing API key" },
+      {
+        status: 401,
+      }
+    );
+  }
+
+  //parse the request body
+  try {
+    const anyBody = await request.json();
+    const body = SendEventBodySchema.safeParse(anyBody);
+    if (!body.success) {
+      return json(
+        { error: generateErrorMessage(body.error.issues) },
+        {
+          status: 422,
+        }
+      );
+    }
+
+    // The AWS SDK tries to use crypto from off of the window,
+    // so we need to trick it into finding it where it expects it
+    globalThis.global = globalThis;
+
+    const client = new SQSClient({
+      region: env.AWS_SQS_REGION,
+      credentials: {
+        accessKeyId: env.AWS_SQS_ACCESS_KEY_ID,
+        secretAccessKey: env.AWS_SQS_SECRET_ACCESS_KEY,
+      },
+    });
+
+    const timestamp = body.data.event.timestamp ?? new Date();
+
+    //add the event to the queue
+    const send = new SendMessageCommand({
+      // use wrangler secrets to provide this global variable
+      QueueUrl: env.AWS_SQS_QUEUE_URL,
+      MessageBody: JSON.stringify({
+        event: { ...body.data.event, timestamp },
+        options: body.data.options,
+        apiKey: apiKeyResult.apiKey,
+      }),
+    });
+
+    const queuedEvent = await client.send(send);
+    console.log("Queued event", queuedEvent);
+
+    //respond with the event
+    const event: ApiEventLog = {
+      id: body.data.event.id,
+      name: body.data.event.name,
+      payload: body.data.event.payload,
+      context: body.data.event.context,
+      timestamp,
+      deliverAt: calculateDeliverAt(body.data.options),
+    };
+
+    return json(event, {
+      status: 200,
+    });
+  } catch (e) {
+    console.error("queueEvent error", e);
+    return json(
+      {
+        error: `Failed to send event: ${e instanceof Error ? e.message : JSON.stringify(e)}`,
+      },
+      {
+        status: 422,
+      }
+    );
+  }
+}

apps/proxy/src/events/queueEvents.ts

@@ -0,0 +1,112 @@
+import { SQSClient, SendMessageBatchCommand } from "@aws-sdk/client-sqs";
+import { ApiEventLog, SendBulkEventsBodySchema } from "@trigger.dev/core";
+import { generateErrorMessage } from "zod-error";
+import { Env } from "..";
+import { getApiKeyFromRequest } from "../apikey";
+import { json } from "../json";
+import { calculateDeliverAt } from "./utils";
+
+/** Adds the event to an AWS SQS queue, so it can be consumed from the main Trigger.dev API */
+export async function queueEvents(request: Request, env: Env): Promise<Response> {
+  //check there's a private API key
+  const apiKeyResult = getApiKeyFromRequest(request);
+  if (!apiKeyResult || apiKeyResult.type !== "PRIVATE") {
+    return json(
+      { error: "Invalid or Missing API key" },
+      {
+        status: 401,
+      }
+    );
+  }
+
+  //parse the request body
+  try {
+    const anyBody = await request.json();
+    const body = SendBulkEventsBodySchema.safeParse(anyBody);
+    if (!body.success) {
+      return json(
+        { error: generateErrorMessage(body.error.issues) },
+        {
+          status: 422,
+        }
+      );
+    }
+
+    // The AWS SDK tries to use crypto from off of the window,
+    // so we need to trick it into finding it where it expects it
+    globalThis.global = globalThis;
+
+    const client = new SQSClient({
+      region: env.AWS_SQS_REGION,
+      credentials: {
+        accessKeyId: env.AWS_SQS_ACCESS_KEY_ID,
+        secretAccessKey: env.AWS_SQS_SECRET_ACCESS_KEY,
+      },
+    });
+
+    const updatedEvents: ApiEventLog[] = body.data.events.map((event) => {
+      const timestamp = event.timestamp ?? new Date();
+      return {
+        ...event,
+        payload: event.payload,
+        timestamp,
+      };
+    });
+
+    //divide updatedEvents into multiple batches of 10 (max size SQS accepts)
+    const batches: ApiEventLog[][] = [];
+    let currentBatch: ApiEventLog[] = [];
+    for (let i = 0; i < updatedEvents.length; i++) {
+      currentBatch.push(updatedEvents[i]);
+      if (currentBatch.length === 10) {
+        batches.push(currentBatch);
+        currentBatch = [];
+      }
+    }
+    if (currentBatch.length > 0) {
+      batches.push(currentBatch);
+    }
+
+    //loop through the batches and send them
+    for (let i = 0; i < batches.length; i++) {
+      const batch = batches[i];
+      //add the event to the queue
+      const send = new SendMessageBatchCommand({
+        // use wrangler secrets to provide this global variable
+        QueueUrl: env.AWS_SQS_QUEUE_URL,
+        Entries: batch.map((event, index) => ({
+          Id: `event-${index}`,
+          MessageBody: JSON.stringify({
+            event,
+            options: body.data.options,
+            apiKey: apiKeyResult.apiKey,
+          }),
+        })),
+      });
+
+      const queuedEvent = await client.send(send);
+      console.log("Queued events", queuedEvent);
+    }
+
+    //respond with the events
+    const events: ApiEventLog[] = updatedEvents.map((event) => ({
+      ...event,
+      payload: event.payload,
+      deliverAt: calculateDeliverAt(body.data.options),
+    }));
+
+    return json(events, {
+      status: 200,
+    });
+  } catch (e) {
+    console.error("queueEvents error", e);
+    return json(
+      {
+        error: `Failed to send events: ${e instanceof Error ? e.message : JSON.stringify(e)}`,
+      },
+      {
+        status: 422,
+      }
+    );
+  }
+}

apps/proxy/src/events/utils.ts

@@ -0,0 +1,15 @@
+import { SendEventOptions } from "@trigger.dev/core";
+
+export function calculateDeliverAt(options?: SendEventOptions) {
+  // If deliverAt is a string and a valid date, convert it to a Date object
+  if (options?.deliverAt) {
+    return options?.deliverAt;
+  }
+
+  // deliverAfter is the number of seconds to wait before delivering the event
+  if (options?.deliverAfter) {
+    return new Date(Date.now() + options.deliverAfter * 1000);
+  }
+
+  return undefined;
+}

apps/proxy/src/index.ts

@@ -0,0 +1,67 @@
+import { queueEvent } from "./events/queueEvent";
+import { queueEvents } from "./events/queueEvents";
+
+export interface Env {
+  /** The hostname needs to be changed to allow requests to pass to the Trigger.dev platform */
+  REWRITE_HOSTNAME: string;
+  REWRITE_PORT?: string;
+  AWS_SQS_ACCESS_KEY_ID: string;
+  AWS_SQS_SECRET_ACCESS_KEY: string;
+  AWS_SQS_QUEUE_URL: string;
+  AWS_SQS_REGION: string;
+}
+
+export default {
+  async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
+    if (!env.REWRITE_HOSTNAME) throw new Error("Missing REWRITE_HOSTNAME");
+    console.log("url", request.url);
+
+    if (!queueingIsEnabled(env)) {
+      console.log("Missing AWS credentials. Passing through to the origin.");
+      return redirectToOrigin(request, env);
+    }
+
+    const url = new URL(request.url);
+    switch (url.pathname) {
+      case "/api/v1/events": {
+        if (request.method === "POST") {
+          return queueEvent(request, env);
+        }
+        break;
+      }
+      case "/api/v1/events/bulk": {
+        if (request.method === "POST") {
+          return queueEvents(request, env);
+        }
+        break;
+      }
+    }
+
+    //the same request but with the hostname (and port) changed
+    return redirectToOrigin(request, env);
+  },
+};
+
+function redirectToOrigin(request: Request, env: Env) {
+  const newUrl = new URL(request.url);
+  newUrl.hostname = env.REWRITE_HOSTNAME;
+  newUrl.port = env.REWRITE_PORT || newUrl.port;
+
+  const requestInit: RequestInit = {
+    method: request.method,
+    headers: request.headers,
+    body: request.body,
+  };
+
+  console.log("rewritten url", newUrl.toString());
+  return fetch(newUrl.toString(), requestInit);
+}
+
+function queueingIsEnabled(env: Env) {
+  return (
+    env.AWS_SQS_ACCESS_KEY_ID &&
+    env.AWS_SQS_SECRET_ACCESS_KEY &&
+    env.AWS_SQS_QUEUE_URL &&
+    env.AWS_SQS_REGION
+  );
+}

apps/proxy/src/json.ts

@@ -0,0 +1,13 @@
+export function json(body: any, init?: ResponseInit) {
+  const headers = {
+    "content-type": "application/json",
+    ...(init?.headers ?? {}),
+  };
+
+  const responseInit: ResponseInit = {
+    ...(init ?? {}),
+    headers,
+  };
+
+  return new Response(JSON.stringify(body), responseInit);
+}

apps/proxy/tsconfig.json

@@ -0,0 +1,35 @@
+{
+  "compilerOptions": {
+    "target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "lib": [
+      "es2021"
+    ] /* Specify a set of bundled library declaration files that describe the target runtime environment. */,
+    "jsx": "react" /* Specify what JSX code is generated. */,
+
+    "module": "es2022" /* Specify what module code is generated. */,
+    "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
+
+    "types": [
+      "@cloudflare/workers-types"
+    ] /* Specify type package names to be included without being referenced in a source file. */,
+    "resolveJsonModule": true /* Enable importing .json files */,
+
+    "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */,
+    "checkJs": false /* Enable error reporting in type-checked JavaScript files. */,
+
+    "noEmit": true /* Disable emitting files from a compilation. */,
+
+    "isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */,
+    "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */,
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+
+    "strict": true /* Enable all strict type-checking options. */,
+
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */,
+    "baseUrl": ".",
+    "paths": {
+      "@trigger.dev/core": ["../../packages/core/src/index"],
+      "@trigger.dev/core/*": ["../../packages/core/src/*"]
+    }
+  }
+}

apps/proxy/wrangler.toml

@@ -0,0 +1,7 @@
+name = "proxy"
+main = "src/index.ts"
+compatibility_date = "2023-10-30"
+compatibility_flags = [ "nodejs_compat" ]
+
+[env.staging]
+[env.prod]

apps/webapp/app/components/integrations/ConnectToIntegrationSheet.tsx

@@ -1,6 +1,6 @@
 import React, { useState } from "react";
-import { ApiAuthenticationMethodApiKey, Integration } from "~/services/externalApis/types";
-import { docsIntegrationPath } from "~/utils/pathBuilder";
+import { Integration } from "~/services/externalApis/types";
+import { apiReferencePath, docsIntegrationPath } from "~/utils/pathBuilder";
 import { LinkButton } from "../primitives/Buttons";
 import { Header1, Header2 } from "../primitives/Headers";
 import { NamedIconInBox } from "../primitives/NamedIcon";
@@ -48,6 +48,15 @@ export function ConnectToIntegrationSheet({
               <Paragraph variant="small">{integration.description}</Paragraph>
             )}
           </div>
+          <LinkButton
+            to={apiReferencePath(integration.identifier)}
+            variant="secondary/small"
+            TrailingIcon="arrow-up-right"
+            trailingIconClassName="h-4 w-4 text-slate-400"
+            target="_blank"
+          >
+            View examples
+          </LinkButton>
           <LinkButton
             to={docsIntegrationPath(integration.identifier)}
             variant="secondary/small"
@@ -83,13 +92,6 @@ export function ConnectToIntegrationSheet({
                 variant="description"
               />
             )}
-            <RadioGroupItem
-              id="custom"
-              value="custom"
-              label="Fetch/Existing SDK"
-              description={`Alternatively, use ${integration.name} without our integration.`}
-              variant="description"
-            />
           </RadioGroup>
 
           {integrationMethod && (
@@ -132,7 +134,5 @@ function SelectedIntegrationMethod({
           callbackUrl={callbackUrl}
         />
       );
-    case "custom":
-      return <CustomHelp name={integration.name} />;
   }
 }

apps/webapp/app/components/integrations/CustomHelp.tsx

@@ -1,79 +1,124 @@
-import { CodeBlock } from "../code/CodeBlock";
+import { useState } from "react";
+import { CodeExample } from "~/routes/resources.codeexample";
+import { Api } from "~/services/externalApis/apis.server";
+import { cn } from "~/utils/cn";
+import { Feedback } from "../Feedback";
 import { Header1, Header2 } from "../primitives/Headers";
 import { Paragraph } from "../primitives/Paragraph";
+import { TextLink } from "../primitives/TextLink";
+
+const fallbackExamples = [
+  {
+    title: "Post to Slack when meetings are booked or cancelled.",
+    slug: "cal-slack-meeting-alert",
+    version: "1.0.0",
+    codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/cal-http-endpoint.ts",
+  },
+  {
+    title: "Translate some text with DeepL.",
+    slug: "translate-text-with-deepl",
+    version: "1.0.0",
+    codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/deepl.ts",
+  },
+  {
+    title: "Create a Discord bot and send a message to a channel.",
+    slug: "discord-bot-send-message",
+    version: "1.0.0",
+    codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/discord.ts",
+  },
+  {
+    title: "Retrieve a Notion page by ID.",
+    slug: "retrieve-notion-page",
+    version: "1.0.0",
+    codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/notion.ts",
+  },
+];
+
+export function CustomHelp({ api }: { api: Api }) {
+  const [selectedExample, setSelectedExample] = useState(0);
+
+  const changeCodeExample = (index: number) => {
+    setSelectedExample(index);
+  };
 
-export function CustomHelp({ name }: { name: string }) {
   return (
     <div className="mt-4">
-      <Header1 className="mb-2">You can use any API with requests or an SDK</Header1>
-      <Header2 className="mb-2">How to use an SDK</Header2>
+      <Header1 className="mb-2">Using an API with an SDK or requests</Header1>
       <Paragraph spacing>
-        You can call SDK methods from inside the run function, but you should wrap them in a Task to
-        make sure they're resumable.
+        You can use Trigger.dev with any existing Node SDK or even just using fetch. You can
+        subscribe to any API with{" "}
+        <TextLink href="https://trigger.dev/docs/documentation/concepts/http-endpoints">
+          HTTP endpoints
+        </TextLink>{" "}
+        and perform actions by wrapping tasks using{" "}
+        <TextLink
+          href="https://trigger.dev/docs/documentation/guides/writing-jobs-step-by-step#create-your-own-tasks"
+          className="font-mono"
+        >
+          io.runTask
+        </TextLink>
+        . This makes your background job resumable and appear in our dashboard.
       </Paragraph>
-      <Paragraph spacing>Here's an example with the official GitHub SDK</Paragraph>
-      <CodeBlock
-        code={`
-client.defineJob({
-  id: "scheduled-job-1",
-  name: "Scheduled Job 1",
-  version: "0.1.1",
-  trigger: cronTrigger({
-    cron: "*/5 * * * *", // every 5 minutes
-  }),
-  run: async (payload, io, ctx) => {
-    //wrap an SDK call in io.runTask so it's resumable and displays in logs
-    const repo = await io.runTask(
-      "Get repo",
-      async () => {
-        //this is the regular GitHub SDK
-        const response = await octokit.rest.repos.get({
-          owner: "triggerdotdev",
-          repo: "trigger.dev",
-        });
-        return response.data;
-      },
-      //you can add metadata to the task to improve the display in the logs
-      { name: "Get repo", icon: "github" }
-    );
-  },
-});
-      `}
-        highlightedRanges={[[9, 22]]}
-        className="mb-4"
-      />
-      <Header2 className="mb-2">How to use fetch</Header2>
-      <Paragraph spacing>
-        You can use the fetch API to make requests to any API. Or a different request library like
-        axios if you'd prefer. Again wrapping the request in a Task will make sure it's resumable.
-      </Paragraph>
-      <CodeBlock
-        code={`
-client.defineJob({
-  id: "scheduled-job-1",
-  name: "Scheduled Job 1",
-  version: "0.1.1",
-  trigger: cronTrigger({
-    cron: "*/5 * * * *", // every 5 minutes
-  }),
-  run: async (payload, io, ctx) => {
-    //wrap anything in io.runTask so it's resumable and displays in logs
-    const repo = await io.runTask(
-      "Get org",
-      async () => {
-        //you can use fetch, axios, or any other library to make requests
-        const response = await fetch('https://api.github.com/orgs/nodejs');
-        return response.json();
-      },
-      //you can add metadata to the task to improve the display in the logs
-      { name: "Get org", icon: "github" }
-    );
-  },
-});
-      `}
-        highlightedRanges={[[9, 19]]}
-        className="mb-4"
-      />
+
+      {api.examples && api.examples.length > 0 ? (
+        <>
+          <Header2 className="mb-2">Example {api.name} code</Header2>
+          <Paragraph spacing className="mb-4">
+            This is how you can use {api.name} with Trigger.dev. This code can be copied and
+            modified to suit your use-case.
+          </Paragraph>
+          {api.examples.length > 1 && (
+            <div className=" flex w-full flex-row gap-4 overflow-x-scroll	scrollbar-thin scrollbar-track-transparent scrollbar-thumb-slate-700">
+              {api.examples?.map((example, index) => (
+                <button
+                  onClick={() => changeCodeExample(index)}
+                  key={example.codeUrl}
+                  className={cn(
+                    "w-64 min-w-[16rem] p-2 transition-colors duration-300 sm:w-full sm:rounded",
+                    "border-px focus:border-px cursor-pointer border border-slate-900 bg-slate-900 text-slate-300 transition duration-300  hover:bg-slate-800 focus:border focus:border-indigo-600"
+                  )}
+                >
+                  {example.title}
+                </button>
+              ))}
+            </div>
+          )}
+          <CodeExample example={api.examples[selectedExample]} />
+        </>
+      ) : (
+        <>
+          <Header2 className="mb-2">Example code using fetch / an existing SDK</Header2>
+          <Paragraph spacing className="mb-4">
+            You can use one of our examples below as a starting point / reference for your projects.
+            Please{" "}
+            <Feedback
+              button={
+                <span className="cursor-pointer text-indigo-500 transition duration-300 hover:text-indigo-400">
+                  reach out to us
+                </span>
+              }
+              defaultValue="help"
+            />{" "}
+            if you're having any issues.
+          </Paragraph>
+
+          <div className=" flex w-full flex-row gap-4 overflow-x-scroll	scrollbar-thin scrollbar-track-transparent scrollbar-thumb-slate-700 sm:h-full">
+            {fallbackExamples.map((example, index) => (
+              <button
+                onClick={() => changeCodeExample(index)}
+                key={example.codeUrl}
+                className={cn(
+                  "w-64 min-w-[16rem] p-2 transition-colors duration-300 sm:w-full sm:rounded",
+                  "border-px focus:border-px cursor-pointer border border-slate-900 bg-slate-900 text-slate-300 transition duration-300 hover:bg-slate-800 focus:border focus:border-indigo-600"
+                )}
+              >
+                {example.title}
+              </button>
+            ))}
+          </div>
+          <CodeExample example={fallbackExamples[selectedExample]} />
+        </>
+      )}
     </div>
   );
 }

apps/webapp/app/components/integrations/NoIntegrationSheet.tsx

@@ -1,14 +1,10 @@
+import { useFetcher } from "@remix-run/react";
 import React from "react";
-import { Api } from "~/services/externalApis/apis";
-import { Button } from "../primitives/Buttons";
-import { Callout } from "../primitives/Callout";
+import { Api } from "~/services/externalApis/apis.server";
 import { Header1 } from "../primitives/Headers";
 import { NamedIconInBox } from "../primitives/NamedIcon";
 import { Sheet, SheetBody, SheetContent, SheetHeader, SheetTrigger } from "../primitives/Sheet";
 import { CustomHelp } from "./CustomHelp";
-import { CheckIcon } from "@heroicons/react/24/solid";
-import { useFetcher } from "@remix-run/react";
-import { Paragraph } from "../primitives/Paragraph";
 
 export function NoIntegrationSheet({
   api,
@@ -31,27 +27,9 @@ export function NoIntegrationSheet({
             <NamedIconInBox name={api.identifier} className="h-9 w-9" />
             <Header1>{api.name}</Header1>
           </div>
-          {requested ? (
-            <div className="flex items-center gap-1">
-              <CheckIcon className="h-4 w-4 text-green-500" />
-              <Paragraph variant="small">
-                We'll let you know when the Integration is available.
-              </Paragraph>
-            </div>
-          ) : (
-            <fetcher.Form method="post" action={`/resources/apivote/${api.identifier}`}>
-              <Button
-                variant="primary/small"
-                disabled={isLoading}
-                LeadingIcon={isLoading ? "spinner-white" : undefined}
-              >
-                {isLoading ? "Saving…" : `I want an Integration for ${api.name}`}
-              </Button>
-            </fetcher.Form>
-          )}
         </SheetHeader>
         <SheetBody>
-          <CustomHelp name={api.name} />
+          <CustomHelp api={api} />
         </SheetBody>
       </SheetContent>
     </Sheet>

apps/webapp/app/components/run/TaskDetail.tsx

@@ -33,7 +33,19 @@ import { Spinner } from "../primitives/Spinner";
 import type { DetailedTask } from "~/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route";
 
 export function TaskDetail({ task }: { task: DetailedTask }) {
-  const { name, description, icon, status, params, properties, output, style, attempts } = task;
+  const {
+    name,
+    description,
+    icon,
+    status,
+    params,
+    properties,
+    output,
+    outputIsUndefined,
+    style,
+    attempts,
+    noop,
+  } = task;
 
   const startedAt = task.startedAt ? new Date(task.startedAt) : undefined;
   const completedAt = task.completedAt ? new Date(task.completedAt) : undefined;
@@ -140,16 +152,18 @@ export function TaskDetail({ task }: { task: DetailedTask }) {
             <Paragraph variant="small">No input</Paragraph>
           )}
         </div>
-        <div className="mt-4 flex flex-col gap-2">
-          <Header3>Output</Header3>
-          {output ? (
-            <ClientOnly fallback={<Spinner />}>
-              {() => <CodeBlock code={output} maxLines={35} />}
-            </ClientOnly>
-          ) : (
-            <Paragraph variant="small">No output</Paragraph>
-          )}
-        </div>
+        {!noop && (
+          <div className="mt-4 flex flex-col gap-2">
+            <Header3>Output</Header3>
+            {output && !outputIsUndefined ? (
+              <ClientOnly fallback={<Spinner />}>
+                {() => <CodeBlock code={output} maxLines={35} />}
+              </ClientOnly>
+            ) : (
+              <Paragraph variant="small">No output</Paragraph>
+            )}
+          </div>
+        )}
       </RunPanelBody>
     </RunPanel>
   );

apps/webapp/app/db.server.ts

@@ -48,7 +48,7 @@ export async function $transaction<R>(
     return await (prisma as PrismaClient).$transaction(fn, options);
   } catch (error) {
     if (isPrismaKnownError(error)) {
-      logger.debug("prisma.$transaction error", {
+      logger.error("prisma.$transaction error", {
         code: error.code,
         meta: error.meta,
         stack: error.stack,

apps/webapp/app/entry.server.tsx

@@ -16,6 +16,8 @@ import {
   OperatingSystemPlatform,
 } from "./components/primitives/OperatingSystemProvider";
 import { env } from "./env.server";
+import { getSharedSqsEventConsumer } from "./services/events/sqsEventConsumer";
+import { singleton } from "./utils/singleton";
 
 const ABORT_DELAY = 30000;
 
@@ -190,3 +192,5 @@ function logError(error: unknown, request?: Request) {
   }
   console.error(error);
 }
+
+const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer);

apps/webapp/app/env.server.ts

@@ -11,7 +11,10 @@ const EnvironmentSchema = z.object({
   SESSION_SECRET: z.string(),
   MAGIC_LINK_SECRET: z.string(),
   ENCRYPTION_KEY: z.string(),
-  WHITELISTED_EMAILS: z.string().refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.").optional(),
+  WHITELISTED_EMAILS: z
+    .string()
+    .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.")
+    .optional(),
   REMIX_APP_PORT: z.string().optional(),
   LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ORIGIN: z.string().default("http://localhost:3030"),
@@ -42,7 +45,20 @@ const EnvironmentSchema = z.object({
   EXECUTION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   WORKER_ENABLED: z.string().default("true"),
   EXECUTION_WORKER_ENABLED: z.string().default("true"),
+  TASK_OPERATION_WORKER_ENABLED: z.string().default("true"),
+  TASK_OPERATION_WORKER_CONCURRENCY: z.coerce.number().int().default(10),
+  TASK_OPERATION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   GRACEFUL_SHUTDOWN_TIMEOUT: z.coerce.number().int().default(60000),
+  /** Optional. Only used if you use the apps/proxy */
+  AWS_SQS_REGION: z.string().optional(),
+  /** Optional. Only used if you use the apps/proxy */
+  AWS_SQS_ACCESS_KEY_ID: z.string().optional(),
+  /** Optional. Only used if you use the apps/proxy */
+  AWS_SQS_SECRET_ACCESS_KEY: z.string().optional(),
+  /** Optional. Only used if you use the apps/proxy */
+  AWS_SQS_QUEUE_URL: z.string().optional(),
+  AWS_SQS_BATCH_SIZE: z.coerce.number().int().optional().default(10),
+  DISABLE_SSE: z.string().optional(),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/hooks/useEventSource.tsx

@@ -0,0 +1,43 @@
+import { useEffect, useState } from "react";
+
+type EventSourceOptions = {
+  init?: EventSourceInit;
+  event?: string;
+  disabled?: boolean;
+};
+
+/**
+ * Subscribe to an event source and return the latest event.
+ * @param url The URL of the event source to connect to
+ * @param options The options to pass to the EventSource constructor
+ * @returns The last event received from the server
+ */
+export function useEventSource(
+  url: string | URL,
+  { event = "message", init, disabled }: EventSourceOptions = {}
+) {
+  const [data, setData] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (disabled) {
+      return;
+    }
+
+    const eventSource = new EventSource(url, init);
+    eventSource.addEventListener(event ?? "message", handler);
+
+    // rest data if dependencies change
+    setData(null);
+
+    function handler(event: MessageEvent) {
+      setData(event.data || "UNKNOWN_EVENT_DATA");
+    }
+
+    return () => {
+      eventSource.removeEventListener(event ?? "message", handler);
+      eventSource.close();
+    };
+  }, [url, event, init, disabled]);
+
+  return data;
+}

apps/webapp/app/hooks/useProjectSetupComplete.ts

@@ -1,9 +1,9 @@
 import { useEffect } from "react";
-import { useEventSource } from "remix-utils/sse/react";
 import { projectPath, projectStreamingPath } from "~/utils/pathBuilder";
 import { useProject } from "./useProject";
 import { useOrganization } from "./useOrganizations";
 import { useNavigate } from "@remix-run/react";
+import { useEventSource } from "./useEventSource";
 
 export function useProjectSetupComplete() {
   const project = useProject();

apps/webapp/app/models/eventDispatcher.server.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+
+export const JobVersionDispatchableSchema = z.object({
+  type: z.literal("JOB_VERSION"),
+  id: z.string(),
+});
+
+export const DynamicTriggerDispatchableSchema = z.object({
+  type: z.literal("DYNAMIC_TRIGGER"),
+  id: z.string(),
+});
+
+export const EphemeralDispatchableSchema = z.object({
+  type: z.literal("EPHEMERAL"),
+  url: z.string(),
+});
+
+export const DispatchableSchema = z.discriminatedUnion("type", [
+  JobVersionDispatchableSchema,
+  DynamicTriggerDispatchableSchema,
+  EphemeralDispatchableSchema,
+]);

apps/webapp/app/models/task.server.ts

@@ -1,7 +1,10 @@
 import type { JobRun, Task, TaskAttempt } from "@trigger.dev/database";
 import { CachedTask, ServerTask } from "@trigger.dev/core";
 
-export type TaskWithAttempts = Task & { attempts: TaskAttempt[]; run: JobRun };
+export type TaskWithAttempts = Task & {
+  attempts: TaskAttempt[];
+  run: { forceYieldImmediately: boolean };
+};
 
 export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask {
   return {
@@ -15,7 +18,8 @@ export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask
     status: task.status,
     description: task.description,
     params: task.params as any,
-    output: task.output as any,
+    output: task.outputIsUndefined ? undefined : (task.output as any),
+    context: task.context as any,
     properties: task.properties as any,
     style: task.style as any,
     error: task.error,
@@ -31,7 +35,7 @@ export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask
 
 export type TaskForCaching = Pick<
   Task,
-  "id" | "status" | "idempotencyKey" | "noop" | "output" | "parentId"
+  "id" | "status" | "idempotencyKey" | "noop" | "output" | "parentId" | "outputIsUndefined"
 >;
 
 export function prepareTasksForCaching(
@@ -104,7 +108,7 @@ function prepareTaskForCaching(task: TaskForCaching): CachedTask {
     status: task.status,
     idempotencyKey: task.idempotencyKey,
     noop: task.noop,
-    output: task.output as any,
+    output: task.outputIsUndefined ? undefined : (task.output as any),
     parentId: task.parentId,
   };
 }

apps/webapp/app/platform/zodWorker.server.ts

@@ -14,8 +14,7 @@ import { run as graphileRun, parseCronItems } from "graphile-worker";
 import omit from "lodash.omit";
 import { z } from "zod";
 import { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
-import { PgListenService } from "~/services/db/pgListen.server";
-import { workerLogger as logger } from "~/services/logger.server";
+import { workerLogger as logger, trace } from "~/services/logger.server";
 
 export interface MessageCatalogSchema {
   [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
@@ -167,21 +166,6 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     this.#runner?.events.on("pool:listen:success", async ({ workerPool, client }) => {
       this.#logDebug("pool:listen:success");
-
-      // hijack client instance to listen and react to incoming NOTIFY events
-      const pgListen = new PgListenService(client, this.#name, logger);
-
-      await pgListen.on("trigger:graphile:migrate", async ({ latestMigration }) => {
-        this.#logDebug("Detected incoming migration", { latestMigration });
-
-        if (latestMigration > 10) {
-          // already migrated past v0.14 - nothing to do
-          return;
-        }
-
-        // simulate SIGTERM to trigger graceful shutdown
-        this._handleSignal("SIGTERM");
-      });
     });
 
     this.#runner?.events.on("pool:listen:error", ({ error }) => {
@@ -286,7 +270,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       spec,
     });
 
-    const job = await this.#addJob(
+    const { job, durationInMs } = await this.#addJob(
       identifier as string,
       payload,
       spec,
@@ -298,6 +282,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       payload,
       spec,
       job,
+      durationInMs,
     });
 
     return job;
@@ -320,6 +305,8 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     spec: TaskSpec,
     tx: PrismaClientOrTransaction
   ) {
+    const now = performance.now();
+
     const results = await tx.$queryRawUnsafe(
       `SELECT * FROM ${this.graphileWorkerSchema}.add_job(
           identifier => $1::text,
@@ -343,6 +330,8 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       spec.jobKeyMode || null
     );
 
+    const durationInMs = performance.now() - now;
+
     const rows = AddJobResultsSchema.safeParse(results);
 
     if (!rows.success) {
@@ -353,7 +342,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     const job = rows.data[0];
 
-    return job as GraphileJob;
+    return { job: job as GraphileJob, durationInMs: Math.floor(durationInMs) };
   }
 
   async #removeJob(jobKey: string, tx: PrismaClientOrTransaction) {
@@ -487,7 +476,15 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       throw new Error(`No task for message type: ${String(typeName)}`);
     }
 
-    await task.handler(payload, job);
+    await trace(
+      {
+        worker_job: job,
+        worker_name: this.#name,
+      },
+      async () => {
+        await task.handler(payload, job);
+      }
+    );
   }
 
   async #handleRecurringTask(

apps/webapp/app/presenters/EnvironmentsStreamPresenter.server.ts

@@ -1,7 +1,7 @@
 import { PrismaClient, prisma } from "~/db.server";
 import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
-import { sse } from "~/utils/sse";
+import { sse } from "~/utils/sse.server";
 
 type EnvironmentSignalsMap = {
   [x: string]: {

apps/webapp/app/presenters/IntegrationsPresenter.server.ts

@@ -3,7 +3,7 @@ import { PrismaClient, prisma } from "~/db.server";
 import { env } from "~/env.server";
 import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
-import { Api, apisList } from "~/services/externalApis/apis";
+import { Api, apisList } from "~/services/externalApis/apis.server";
 import { integrationCatalog } from "~/services/externalApis/integrationCatalog.server";
 import { Integration, OAuthClientSchema } from "~/services/externalApis/types";
 import { getSecretStore } from "~/services/secrets/secretStore.server";

apps/webapp/app/presenters/RunStreamPresenter.server.ts

@@ -1,6 +1,6 @@
 import { JobRun } from "@trigger.dev/database";
 import { PrismaClient, prisma } from "~/db.server";
-import { sse } from "~/utils/sse";
+import { sse } from "~/utils/sse.server";
 
 export class RunStreamPresenter {
   #prismaClient: PrismaClient;

apps/webapp/app/presenters/TaskDetailsPresenter.server.ts

@@ -1,5 +1,4 @@
-import { RedactSchema } from "@trigger.dev/core";
-import { StyleSchema } from "@trigger.dev/core";
+import { RedactSchema, StyleSchema } from "@trigger.dev/core";
 import { PrismaClient, prisma } from "~/db.server";
 import { mergeProperties } from "~/utils/mergeProperties.server";
 import { Redactor } from "~/utils/redactor";
@@ -58,6 +57,7 @@ export class TaskDetailsPresenter {
         outputProperties: true,
         params: true,
         output: true,
+        outputIsUndefined: true,
         error: true,
         startedAt: true,
         completedAt: true,
@@ -89,9 +89,11 @@ export class TaskDetailsPresenter {
     return {
       ...task,
       redact: undefined,
-      output: task.output
-        ? JSON.stringify(this.#stringifyOutputWithRedactions(task.output, task.redact), null, 2)
-        : undefined,
+      output: JSON.stringify(
+        this.#stringifyOutputWithRedactions(task.output, task.redact),
+        null,
+        2
+      ),
       connection: task.runConnection,
       params: task.params as Record<string, any>,
       properties: mergeProperties(task.properties, task.outputProperties),
@@ -101,7 +103,7 @@ export class TaskDetailsPresenter {
 
   #stringifyOutputWithRedactions(output: any, redact: unknown): any {
     if (!output) {
-      return;
+      return output;
     }
 
     const parsedRedact = RedactSchema.safeParse(redact);

apps/webapp/app/routes/_app._index/route.tsx

@@ -1,19 +1,26 @@
 import { LoaderFunctionArgs, redirect } from "@remix-run/server-runtime";
+import { getUsersInvites } from "~/models/member.server";
 import { SelectBestProjectPresenter } from "~/presenters/SelectBestProjectPresenter.server";
-import { requireUserId } from "~/services/session.server";
-import { newOrganizationPath, projectPath } from "~/utils/pathBuilder";
+import { requireUser } from "~/services/session.server";
+import { invitesPath, newOrganizationPath, projectPath } from "~/utils/pathBuilder";
 
 //this loader chooses the best project to redirect you to, ideally based on the cookie
 export const loader = async ({ request }: LoaderFunctionArgs) => {
-  const userId = await requireUserId(request);
+  const user = await requireUser(request);
+
+  //if there are invites then we should redirect to the invites page
+  const invites = await getUsersInvites({ email: user.email });
+  if (invites.length > 0) {
+    return redirect(invitesPath());
+  }
 
   const presenter = new SelectBestProjectPresenter();
   try {
-    const { project, organization } = await presenter.call({ userId, request });
-
+    const { project, organization } = await presenter.call({ userId: user.id, request });
+    //redirect them to the most appropriate project
     return redirect(projectPath(organization, project));
   } catch (e) {
-    //this should only happen if the user has no projects
+    //this should only happen if the user has no projects, and no invites
     return redirect(newOrganizationPath());
   }
 };

apps/webapp/app/routes/_app.orgs.$organizationSlug.integrations/route.tsx

@@ -82,7 +82,7 @@ export default function Integrations() {
     <PageContainer>
       <PageHeader>
         <PageTitleRow>
-          <PageTitle title="Integrations" />
+          <PageTitle title="Integrations & APIs" />
           <PageButtons>
             <LinkButton
               to={docsPath("/integrations/introduction")}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx

@@ -2,7 +2,7 @@ import { conform, useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
 import { useFetcher, useRevalidator } from "@remix-run/react";
 import { useEffect } from "react";
-import { useEventSource } from "remix-utils/sse/react";
+import { useEventSource } from "~/hooks/useEventSource";
 import { InlineCode } from "~/components/code/InlineCode";
 import {
   EndpointIndexStatusIcon,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/FirstEndpointSheet.tsx

@@ -51,7 +51,7 @@ export function FirstEndpointSheet({ projectId, environments }: FirstEndpointShe
   return (
     <Sheet>
       <SheetTrigger>
-        <Button variant="secondary/medium">Add your first endpoint</Button>
+        <ButtonContent variant="secondary/medium">Add your first endpoint</ButtonContent>
       </SheetTrigger>
       <SheetContent size="lg">
         <SheetHeader>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/route.tsx

@@ -2,7 +2,7 @@ import { useRevalidator } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { useEffect, useMemo, useState } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
-import { useEventSource } from "remix-utils/sse/react";
+import { useEventSource } from "~/hooks/useEventSource";
 import {
   EndpointIndexStatusIcon,
   EndpointIndexStatusLabel,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam.tasks.$taskParam/route.tsx

@@ -1,6 +1,5 @@
-import { Await, useLoaderData } from "@remix-run/react";
-import { LoaderFunctionArgs, SerializeFrom, defer } from "@remix-run/server-runtime";
-import { Suspense } from "react";
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { Spinner } from "~/components/primitives/Spinner";
 import { TaskDetail } from "~/components/run/TaskDetail";
 import { TaskDetailsPresenter } from "~/presenters/TaskDetailsPresenter.server";
@@ -12,26 +11,28 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const { taskParam } = TaskParamsSchema.parse(params);
 
   const presenter = new TaskDetailsPresenter();
-  const taskPromise = presenter.call({
+  const task = await presenter.call({
     userId,
     id: taskParam,
   });
 
-  return defer({
-    taskPromise,
+  return typedjson({
+    task,
   });
 };
 
-export type DetailedTask = NonNullable<Awaited<SerializeFrom<typeof loader>["taskPromise"]>>;
+export type DetailedTask = NonNullable<UseDataFunctionReturn<typeof loader>["task"]>;
 
 export default function Page() {
-  const { taskPromise } = useLoaderData<typeof loader>();
+  const { task } = useTypedLoaderData<typeof loader>();
 
-  return (
-    <Suspense fallback={<Spinner />}>
-      <Await resolve={taskPromise} errorElement={<p>Error loading task!</p>}>
-        {(resolvedTask) => resolvedTask && <TaskDetail task={resolvedTask as any} />}
-      </Await>
-    </Suspense>
-  );
+  if (!task) {
+    return (
+      <div className="flex h-full items-center justify-center">
+        <Spinner />
+      </div>
+    );
+  }
+
+  return <TaskDetail task={task} />;
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam/route.tsx

@@ -2,7 +2,7 @@ import { useRevalidator } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { Fragment, useEffect } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
-import { useEventSource } from "remix-utils/sse/react";
+import { useEventSource } from "~/hooks/useEventSource";
 import { BreadcrumbLink } from "~/components/navigation/Breadcrumb";
 import { BreadcrumbIcon } from "~/components/primitives/BreadcrumbIcon";
 import { RunOverview } from "~/components/run/RunOverview";
@@ -67,6 +67,7 @@ export default function Page() {
   const revalidator = useRevalidator();
   const events = useEventSource(runStreamingPath(organization, project, job, run), {
     event: "message",
+    disabled: !!run.completedAt,
   });
   useEffect(() => {
     if (events !== null) {

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.test/route.tsx

@@ -1,6 +1,5 @@
 import { useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
-import { ClipboardIcon } from "@heroicons/react/20/solid";
 import { ClockIcon, CodeBracketIcon } from "@heroicons/react/24/outline";
 import { Form, useActionData, useSubmit } from "@remix-run/react";
 import { ActionFunction, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
@@ -33,10 +32,14 @@ import { redirectBackWithErrorMessage, redirectWithSuccessMessage } from "~/mode
 import { TestJobPresenter } from "~/presenters/TestJobPresenter.server";
 import { TestJobService } from "~/services/jobs/testJob.server";
 import { requireUserId } from "~/services/session.server";
-import { cn } from "~/utils/cn";
 import { Handle } from "~/utils/handle";
 import { isValidIcon } from "~/utils/icon";
-import { JobParamsSchema, jobRunDashboardPath, trimTrailingSlash } from "~/utils/pathBuilder";
+import {
+  JobParamsSchema,
+  docsPath,
+  jobRunDashboardPath,
+  trimTrailingSlash,
+} from "~/utils/pathBuilder";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -139,7 +142,7 @@ export default function Page() {
     setDefaultJson(code);
   }, []);
 
-  const [selectedEnvironmentId, setSelectedEnvironmentId] = useState<string>(environments[0].id);
+  const [selectedEnvironmentId, setSelectedEnvironmentId] = useState(environments.at(0)?.id);
   const selectedEnvironment = environments.find((e) => e.id === selectedEnvironmentId);
 
   const currentJson = useRef<string>(defaultJson);
@@ -147,6 +150,10 @@ export default function Page() {
 
   const submitForm = useCallback(
     (e: React.FormEvent<HTMLFormElement>) => {
+      if (!selectedEnvironmentId) {
+        return;
+      }
+
       submit(
         {
           payload: currentJson.current,
@@ -175,10 +182,33 @@ export default function Page() {
 
   if (environments.length === 0) {
     return (
-      <Callout variant="warning">
-        Can't run a test when there are no environments. This shouldn't happen, please contact
-        support.
-      </Callout>
+      <div className="flex flex-col gap-4">
+        <Callout variant="info">
+          There are no environments that you can test this job with – you can't run Tests against
+          your teammates' Dev environments. You should run the code locally (using the CLI) so that
+          this Job will be associated with your Dev environment. This also means that this Job
+          hasn't been deployed to Staging or Prod yet.
+        </Callout>
+        <div>
+          <Header2 spacing>Useful guides</Header2>
+          <div className="flex gap-2">
+            <LinkButton
+              to={docsPath("documentation/guides/cli#dev-command")}
+              variant="secondary/small"
+              LeadingIcon="docs"
+            >
+              Using the CLI
+            </LinkButton>
+            <LinkButton
+              to={docsPath("documentation/guides/deployment")}
+              variant="secondary/small"
+              LeadingIcon="docs"
+            >
+              Deploying your Jobs
+            </LinkButton>
+          </div>
+        </div>
+      </div>
     );
   }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam_.runs.$runParam.tasks.$taskParam/route.tsx

@@ -1,6 +1,5 @@
-import { Await, useLoaderData } from "@remix-run/react";
-import { LoaderFunctionArgs, defer } from "@remix-run/server-runtime";
-import { Suspense } from "react";
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { Spinner } from "~/components/primitives/Spinner";
 import { TaskDetail } from "~/components/run/TaskDetail";
 import { TaskDetailsPresenter } from "~/presenters/TaskDetailsPresenter.server";
@@ -12,24 +11,26 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const { taskParam } = TriggerSourceRunTaskParamsSchema.parse(params);
 
   const presenter = new TaskDetailsPresenter();
-  const taskPromise = presenter.call({
+  const task = await presenter.call({
     userId,
     id: taskParam,
   });
 
-  return defer({
-    taskPromise,
+  return typedjson({
+    task,
   });
 };
 
 export default function Page() {
-  const { taskPromise } = useLoaderData<typeof loader>();
+  const { task } = useTypedLoaderData<typeof loader>();
 
-  return (
-    <Suspense fallback={<Spinner />}>
-      <Await resolve={taskPromise} errorElement={<p>Error loading task!</p>}>
-        {(resolvedTask) => resolvedTask && <TaskDetail task={resolvedTask as any} />}
-      </Await>
-    </Suspense>
-  );
+  if (!task) {
+    return (
+      <div className="flex h-full items-center justify-center">
+        <Spinner />
+      </div>
+    );
+  }
+
+  return <TaskDetail task={task} />;
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers_.external.$triggerParam_.runs.$runParam/route.tsx

@@ -2,7 +2,7 @@ import { useRevalidator } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { Fragment, useEffect } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
-import { useEventSource } from "remix-utils/sse/react";
+import { useEventSource } from "~/hooks/useEventSource";
 import { BreadcrumbLink } from "~/components/navigation/Breadcrumb";
 import { BreadcrumbIcon } from "~/components/primitives/BreadcrumbIcon";
 import { RunOverview } from "~/components/run/RunOverview";

apps/webapp/app/routes/admin.api.v1.snapshot.ts

@@ -0,0 +1,59 @@
+import path from "path";
+import os from "os";
+import fs from "fs";
+import v8 from "v8";
+import { PassThrough } from "stream";
+import { json, type DataFunctionArgs } from "@remix-run/node";
+import { prisma } from "~/db.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { requireUser } from "~/services/session.server";
+
+// Format date as yyyy-MM-dd HH_mm_ss_SSS
+function formatDate(date: Date) {
+  const year = date.getFullYear();
+  const month = date.getMonth() + 1;
+  const day = date.getDate();
+  const hours = date.getHours();
+  const minutes = date.getMinutes();
+  const seconds = date.getSeconds();
+  const milliseconds = date.getMilliseconds();
+
+  return `${year}-${month.toString().padStart(2, "0")}-${day.toString().padStart(2, "0")} ${hours
+    .toString()
+    .padStart(2, "0")}_${minutes.toString().padStart(2, "0")}_${seconds
+    .toString()
+    .padStart(2, "0")}_${milliseconds.toString().padStart(3, "0")}`;
+}
+
+export async function loader({ request }: DataFunctionArgs) {
+  const user = await requireUser(request);
+
+  if (!user.admin) {
+    throw new Response("You must be an admin to perform this action", { status: 403 });
+  }
+
+  const host = request.headers.get("X-Forwarded-Host") ?? request.headers.get("host");
+
+  const tempDir = os.tmpdir();
+  const filepath = path.join(tempDir, `${host}-${formatDate(new Date())}.heapsnapshot`);
+
+  const snapshotPath = v8.writeHeapSnapshot(filepath);
+  if (!snapshotPath) {
+    throw new Response("No snapshot saved", { status: 500 });
+  }
+
+  const body = new PassThrough();
+  const stream = fs.createReadStream(snapshotPath);
+  stream.on("open", () => stream.pipe(body));
+  stream.on("error", (err) => body.end(err));
+  stream.on("end", () => body.end());
+
+  return new Response(body as any, {
+    status: 200,
+    headers: {
+      "Content-Type": "application/octet-stream",
+      "Content-Disposition": `attachment; filename="${path.basename(snapshotPath)}"`,
+      "Content-Length": (await fs.promises.stat(snapshotPath)).size.toString(),
+    },
+  });
+}

apps/webapp/app/routes/api.v1.event-dispatchers.ephemeral.ts

@@ -0,0 +1,59 @@
+import type { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import {
+  EphemeralEventDispatcherRequestBodySchema,
+  InvokeJobRequestBodySchema,
+} from "@trigger.dev/core";
+import { z } from "zod";
+import { PrismaErrorSchema } from "~/db.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CreateEphemeralEventDispatcherService } from "~/services/dispatchers/createEphemeralEventDispatcher.server";
+import { InvokeJobService } from "~/services/jobs/invokeJob.server";
+import { logger } from "~/services/logger.server";
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API Key" }, { status: 401 });
+  }
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  logger.debug("CreateEphemeralEventDispatcherService.call() request body", {
+    body: anyBody,
+  });
+
+  const body = EphemeralEventDispatcherRequestBodySchema.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new CreateEphemeralEventDispatcherService();
+
+  try {
+    const dispatcher = await service.call(authenticationResult.environment, body.data);
+
+    if (!dispatcher) {
+      return json({ error: "Could not create Event Dispatcher" }, { status: 500 });
+    }
+
+    return json({ id: dispatcher.id });
+  } catch (error) {
+    const prismaError = PrismaErrorSchema.safeParse(error);
+    // Record not found in the database
+    if (prismaError.success && prismaError.data.code === "P2005") {
+      return json({ error: "Dispatcher not found" }, { status: 404 });
+    } else {
+      return json({ error: "Internal Server Error" }, { status: 500 });
+    }
+  }
+}

apps/webapp/app/routes/api.v1.events.bulk.ts

@@ -0,0 +1,49 @@
+import type { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { SendBulkEventsBodySchema } from "@trigger.dev/core";
+import { generateErrorMessage } from "zod-error";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { IngestSendEvent } from "~/services/events/ingestSendEvent.server";
+import { eventRecordToApiJson } from "~/api.server";
+import { EventRecord } from "@trigger.dev/database";
+
+export async function action({ request }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  const body = SendBulkEventsBodySchema.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ message: generateErrorMessage(body.error.issues) }, { status: 422 });
+  }
+
+  const service = new IngestSendEvent();
+
+  const events: EventRecord[] = [];
+
+  for (const event of body.data.events) {
+    const eventRecord = await service.call(authenticatedEnv, event, body.data.options);
+
+    if (!eventRecord) {
+      return json({ error: "Failed to create event during bulk ingest" }, { status: 500 });
+    }
+
+    events.push(eventRecord);
+  }
+
+  return json(events.map(eventRecordToApiJson));
+}

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete.ts

@@ -191,7 +191,8 @@ export class CompleteRunTaskService {
       },
       data: {
         status: "COMPLETED",
-        output: taskBody.output ?? undefined,
+        output: taskBody.output as any,
+        outputIsUndefined: typeof taskBody.output === "undefined",
         completedAt: new Date(),
         outputProperties: taskBody.properties,
       },

apps/webapp/app/routes/api.v1.runs.$runId.tasks.ts

@@ -1,22 +1,17 @@
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import { TaskStatus } from "@trigger.dev/database";
 import {
   API_VERSIONS,
-  RunTaskBodyOutput,
   RunTaskBodyOutputSchema,
   RunTaskResponseWithCachedTasksBody,
   ServerTask,
 } from "@trigger.dev/core";
 import { z } from "zod";
-import { $transaction, PrismaClient, prisma } from "~/db.server";
-import { prepareTasksForCaching, taskWithAttemptsToServerTask } from "~/models/task.server";
+import { PrismaClient, prisma } from "~/db.server";
+import { prepareTasksForCaching } from "~/models/task.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { ulid } from "~/services/ulid.server";
-import { workerQueue } from "~/services/worker.server";
-import { generateSecret } from "~/services/sources/utils.server";
-import { env } from "~/env.server";
+import { RunTaskService } from "~/services/tasks/runTask.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -161,184 +156,3 @@ class ChangeRequestLazyLoadedCachedTasks {
     };
   }
 }
-
-export class RunTaskService {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
-  public async call(
-    runId: string,
-    idempotencyKey: string,
-    taskBody: RunTaskBodyOutput
-  ): Promise<ServerTask | undefined> {
-    const task = await $transaction(this.#prismaClient, async (tx) => {
-      const existingTask = await tx.task.findUnique({
-        where: {
-          runId_idempotencyKey: {
-            runId,
-            idempotencyKey,
-          },
-        },
-        include: {
-          attempts: true,
-          run: true,
-        },
-      });
-
-      const delayUntilInFuture = taskBody.delayUntil && taskBody.delayUntil.getTime() > Date.now();
-      const callbackEnabled = taskBody.callback?.enabled;
-
-      if (existingTask) {
-        if (existingTask.status === "CANCELED") {
-          const existingTaskStatus =
-            delayUntilInFuture || callbackEnabled || taskBody.trigger
-              ? "WAITING"
-              : taskBody.noop
-              ? "COMPLETED"
-              : "RUNNING";
-
-          const resumedExistingTask = await tx.task.update({
-            where: {
-              id: existingTask.id,
-            },
-            data: {
-              status: existingTaskStatus,
-              startedAt: new Date(),
-              completedAt: existingTaskStatus === "COMPLETED" ? new Date() : undefined,
-            },
-            include: {
-              run: true,
-              attempts: true,
-            },
-          });
-
-          return resumedExistingTask;
-        }
-
-        return existingTask;
-      }
-
-      const run = await tx.jobRun.findUnique({
-        where: {
-          id: runId,
-        },
-        select: {
-          status: true,
-        },
-      });
-
-      if (!run) throw new Error("Run not found");
-
-      // If task.delayUntil is set and is in the future, we'll set the task's status to "WAITING", else set it to RUNNING
-      let status: TaskStatus;
-
-      if (run.status === "CANCELED") {
-        status = "CANCELED";
-      } else {
-        status =
-          delayUntilInFuture || callbackEnabled || taskBody.trigger
-            ? "WAITING"
-            : taskBody.noop
-            ? "COMPLETED"
-            : "RUNNING";
-      }
-
-      const taskId = ulid();
-      const callbackUrl = callbackEnabled
-        ? `${env.APP_ORIGIN}/api/v1/tasks/${taskId}/callback/${generateSecret(12)}`
-        : undefined;
-
-      const task = await tx.task.create({
-        data: {
-          id: taskId,
-          idempotencyKey,
-          displayKey: taskBody.displayKey,
-          runConnection: taskBody.connectionKey
-            ? {
-                connect: {
-                  runId_key: {
-                    runId,
-                    key: taskBody.connectionKey,
-                  },
-                },
-              }
-            : undefined,
-          icon: taskBody.icon,
-          run: {
-            connect: {
-              id: runId,
-            },
-          },
-          parent: taskBody.parentId ? { connect: { id: taskBody.parentId } } : undefined,
-          name: taskBody.name ?? "Task",
-          description: taskBody.description,
-          status,
-          startedAt: new Date(),
-          completedAt: status === "COMPLETED" || status === "CANCELED" ? new Date() : undefined,
-          noop: taskBody.noop,
-          delayUntil: taskBody.delayUntil,
-          params: taskBody.params ?? undefined,
-          properties: this.#filterProperties(taskBody.properties) ?? undefined,
-          redact: taskBody.redact ?? undefined,
-          operation: taskBody.operation,
-          callbackUrl,
-          style: taskBody.style ?? { style: "normal" },
-          childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
-          attempts: {
-            create: {
-              number: 1,
-              status: "PENDING",
-            },
-          },
-        },
-        include: {
-          run: true,
-          attempts: true,
-        },
-      });
-
-      if (task.status === "RUNNING" && typeof taskBody.operation === "string") {
-        // We need to schedule the operation
-        await workerQueue.enqueue(
-          "performTaskOperation",
-          {
-            id: task.id,
-          },
-          { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
-        );
-      } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
-        if (taskBody.callback.timeoutInSeconds > 0) {
-          // We need to schedule the callback timeout
-          await workerQueue.enqueue(
-            "processCallbackTimeout",
-            {
-              id: task.id,
-            },
-            {
-              tx,
-              runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
-              jobKey: `process-callback:${task.id}`,
-            }
-          );
-        }
-      }
-
-      return task;
-    });
-
-    return task ? taskWithAttemptsToServerTask(task) : undefined;
-  }
-
-  #filterProperties(properties: RunTaskBodyOutput["properties"]): RunTaskBodyOutput["properties"] {
-    if (!properties) return;
-
-    return properties.filter((property) => {
-      if (!property) return false;
-
-      return typeof property.label === "string" && typeof property.text === "string";
-    });
-  }
-}

apps/webapp/app/routes/resources.codeexample.tsx

@@ -0,0 +1,49 @@
+import { useFetcher } from "@remix-run/react";
+import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { useEffect } from "react";
+import invariant from "tiny-invariant";
+import { CodeBlock } from "~/components/code/CodeBlock";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { Spinner } from "~/components/primitives/Spinner";
+import { ApiExample } from "~/services/externalApis/apis.server";
+import { requireUser } from "~/services/session.server";
+export async function loader({ request }: LoaderFunctionArgs) {
+  await requireUser(request);
+  const url = new URL(request.url);
+  const codeUrl = url.searchParams.get("url");
+  invariant(typeof codeUrl === "string", "codeUrl is required");
+  const decodedCodeUrl = decodeURIComponent(codeUrl);
+  const response = await fetch(decodedCodeUrl);
+  if (!response.ok) {
+    throw new Error("Network response was not ok");
+  }
+
+  const code = await response.text();
+
+  const hideCodeRegex = /(\n)?\/\/ hide-code[\s\S]*?\/\/ end-hide-code(\n)*/gm;
+  const cleanedCode = code?.replace(hideCodeRegex, "\n");
+
+  return json({
+    code: cleanedCode,
+  });
+}
+
+export function CodeExample({ example }: { example: ApiExample }) {
+  const customerFetcher = useFetcher<typeof loader>();
+
+  useEffect(() => {
+    customerFetcher.load(`/resources/codeexample?url=${encodeURIComponent(example.codeUrl)}`);
+  }, [example.codeUrl]);
+
+  if (customerFetcher.state === "loading")
+    return (
+      <div className="flex h-96 w-full items-center justify-center gap-2 rounded-md border border-slate-800 font-mono">
+        <Spinner />
+        <Paragraph>Loading example code</Paragraph>
+      </div>
+    );
+
+  return (
+    customerFetcher.data && <CodeBlock code={customerFetcher.data.code ?? ""} className="mt-2" />
+  );
+}

apps/webapp/app/routes/resources.environments.$environmentParam.endpoint.stream.ts

@@ -2,7 +2,7 @@ import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { requireUserId } from "~/services/session.server";
-import { sse } from "~/utils/sse";
+import { sse } from "~/utils/sse.server";
 
 export async function loader({ request, params }: LoaderFunctionArgs) {
   await requireUserId(request);

apps/webapp/app/routes/resources.projects.$projectId.jobs.stream.ts

@@ -2,7 +2,7 @@ import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { requireUserId } from "~/services/session.server";
-import { sse } from "~/utils/sse";
+import { sse } from "~/utils/sse.server";
 
 export async function loader({ request, params }: LoaderFunctionArgs) {
   await requireUserId(request);

apps/webapp/app/routes/tests.sse.stream.ts

@@ -1,7 +1,7 @@
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { logger } from "~/services/logger.server";
-import { sse } from "~/utils/sse";
+import { sse } from "~/utils/sse.server";
 
 export async function loader({ request }: LoaderFunctionArgs) {
   const url = new URL(request.url);

apps/webapp/app/routes/tests.sse.tsx

@@ -1,6 +1,6 @@
 import { useLoaderData } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { useEventSource } from "remix-utils/sse/react";
+import { useEventSource } from "~/hooks/useEventSource";
 import { z } from "zod";
 
 export async function loader({ request }: LoaderFunctionArgs) {

apps/webapp/app/services/apiAuth.server.ts

@@ -20,7 +20,19 @@ export async function authenticateApiRequest(
   request: Request,
   { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
 ): Promise<ApiAuthenticationResult | undefined> {
-  const result = getApiKeyFromRequest(request);
+  const apiKey = getApiKeyFromRequest(request);
+  if (!apiKey) {
+    return;
+  }
+
+  return authenticateApiKey(apiKey, { allowPublicKey });
+}
+
+export async function authenticateApiKey(
+  apiKey: string,
+  { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
+): Promise<ApiAuthenticationResult | undefined> {
+  const result = getApiKeyResult(apiKey);
 
   if (!result) {
     return;
@@ -69,6 +81,10 @@ export function getApiKeyFromRequest(request: Request) {
   }
 
   const apiKey = authorization.data.replace(/^Bearer /, "");
+  return apiKey;
+}
+
+export function getApiKeyResult(apiKey: string) {
   const type = isPublicApiKey(apiKey) ? ("PUBLIC" as const) : ("PRIVATE" as const);
   return { apiKey, type };
 }

apps/webapp/app/services/dispatchers/createEphemeralEventDispatcher.server.ts

@@ -0,0 +1,68 @@
+import { EphemeralEventDispatcherRequestBody } from "@trigger.dev/core";
+import { $transaction, PrismaClient, prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+import { ExpireDispatcherService } from "./expireDispatcher.server";
+
+export class CreateEphemeralEventDispatcherService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    environment: AuthenticatedEnvironment,
+    data: EphemeralEventDispatcherRequestBody
+  ) {
+    return await $transaction(this.#prismaClient, async (tx) => {
+      const existingDispatcher = await tx.eventDispatcher.findUnique({
+        where: {
+          dispatchableId_environmentId: {
+            dispatchableId: data.url,
+            environmentId: environment.id,
+          },
+        },
+      });
+
+      if (existingDispatcher) {
+        return existingDispatcher;
+      }
+
+      const externalAccount = data.accountId
+        ? await this.#prismaClient.externalAccount.upsert({
+            where: {
+              environmentId_identifier: {
+                environmentId: environment.id,
+                identifier: data.accountId,
+              },
+            },
+            create: {
+              environmentId: environment.id,
+              organizationId: environment.organizationId,
+              identifier: data.accountId,
+            },
+            update: {},
+          })
+        : undefined;
+
+      const dispatcher = await tx.eventDispatcher.create({
+        data: {
+          dispatchableId: data.url,
+          environmentId: environment.id,
+          source: data.source ?? "trigger.dev",
+          payloadFilter: data.filter,
+          contextFilter: data.contextFilter,
+          dispatchable: { url: data.url, type: "EPHEMERAL" },
+          enabled: true,
+          event: typeof data.name === "string" ? [data.name] : data.name,
+          manual: false,
+          externalAccountId: externalAccount?.id,
+        },
+      });
+
+      await ExpireDispatcherService.enqueue(dispatcher.id, data.timeoutInSeconds, tx);
+
+      return dispatcher;
+    });
+  }
+}

apps/webapp/app/services/dispatchers/expireDispatcher.server.ts

@@ -0,0 +1,36 @@
+import { PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+
+export class ExpireDispatcherService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    await this.#prismaClient.eventDispatcher.delete({
+      where: {
+        id,
+      },
+    });
+  }
+
+  static async dequeue(id: string, tx?: PrismaClientOrTransaction) {
+    await workerQueue.dequeue(`expire:${id}`, { tx });
+  }
+
+  static async enqueue(id: string, timeoutInSeconds: number, tx?: PrismaClientOrTransaction) {
+    await workerQueue.enqueue(
+      "expireDispatcher",
+      {
+        id,
+      },
+      {
+        tx,
+        runAt: new Date(Date.now() + 1000 * timeoutInSeconds),
+        jobKey: `expire:${id}`,
+      }
+    );
+  }
+}

apps/webapp/app/services/dispatchers/invokeEphemeralEventDispatcher.server.ts

@@ -0,0 +1,98 @@
+import { PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
+import { taskOperationWorker } from "../worker.server";
+import { EphemeralDispatchableSchema } from "~/models/eventDispatcher.server";
+import { ExpireDispatcherService } from "./expireDispatcher.server";
+
+export class InvokeEphemeralDispatcherService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string, eventRecordId: string) {
+    const eventDispatcher = await this.#prismaClient.eventDispatcher.findUnique({
+      where: {
+        id,
+      },
+    });
+
+    if (!eventDispatcher) {
+      return;
+    }
+
+    if (!eventDispatcher.enabled) {
+      return;
+    }
+
+    const eventRecord = await this.#prismaClient.eventRecord.findUnique({
+      where: {
+        id: eventRecordId,
+      },
+      include: {
+        externalAccount: true,
+      },
+    });
+
+    if (!eventRecord) {
+      return;
+    }
+
+    if (eventRecord.cancelledAt) {
+      return;
+    }
+
+    const dispatchable = EphemeralDispatchableSchema.safeParse(eventDispatcher.dispatchable);
+
+    if (!dispatchable.success) {
+      return;
+    }
+
+    const url = dispatchable.data.url;
+
+    const body = {
+      id: eventRecord.eventId,
+      source: eventRecord.source,
+      name: eventRecord.name,
+      payload: eventRecord.payload,
+      context: eventRecord.context,
+      timestamp: eventRecord.timestamp,
+      accountId: eventRecord.externalAccount ? eventRecord.externalAccount.identifier : undefined,
+    };
+
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json; charset=utf-8",
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (!response.ok) {
+      throw new Error(
+        `Failed to invoke ephemeral dispatcher: ${response.statusText} [${response.status}]`
+      );
+    }
+
+    // Run the expire dispatcher service
+    await ExpireDispatcherService.enqueue(id, 0);
+  }
+
+  static async dequeue(id: string, tx?: PrismaClientOrTransaction) {
+    await taskOperationWorker.dequeue(`invoke:ephemeral:${id}`, { tx });
+  }
+
+  static async enqueue(id: string, eventRecordId: string, tx?: PrismaClientOrTransaction) {
+    await taskOperationWorker.enqueue(
+      "invokeEphemeralDispatcher",
+      {
+        id,
+        eventRecordId,
+      },
+      {
+        tx,
+        jobKey: `invoke:ephemeral:${id}`,
+      }
+    );
+  }
+}

apps/webapp/app/services/endpointApi.server.ts

@@ -3,6 +3,7 @@ import {
   ConnectionAuth,
   EndpointHeadersSchema,
   ErrorWithStackSchema,
+  ExecuteJobHeadersSchema,
   HttpSourceResponseSchema,
   IndexEndpointResponseSchema,
   NormalizedResponseSchema,
@@ -14,6 +15,7 @@ import {
   RegisterTriggerBodyV1,
   RunJobBody,
   RunJobResponseSchema,
+  RunNotification,
   ValidateResponse,
   ValidateResponseSchema,
 } from "@trigger.dev/core";
@@ -148,6 +150,7 @@ export class EndpointApi {
       response,
       parser: RunJobResponseSchema,
       errorParser: ErrorWithStackSchema,
+      headersParser: ExecuteJobHeadersSchema,
       durationInMs: Math.floor(performance.now() - startTimeInMs),
     };
   }
@@ -367,6 +370,20 @@ export class EndpointApi {
       durationInMs: Math.floor(performance.now() - startTimeInMs),
     };
   }
+
+  async deliverRunNotification(notification: RunNotification<any>) {
+    const response = await safeFetch(this.url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "x-trigger-api-key": this.apiKey,
+        "x-trigger-action": "RUN_NOTIFICATION",
+      },
+      body: JSON.stringify(notification),
+    });
+
+    return response;
+  }
 }
 
 async function safeFetch(url: string, options: RequestInit) {

apps/webapp/app/services/events/deliverEvent.server.ts

@@ -96,6 +96,13 @@ export class DeliverEventService {
       return true;
     }
 
+    if (
+      dispatcher.externalAccountId &&
+      dispatcher.externalAccountId !== eventRecord.externalAccountId
+    ) {
+      return false;
+    }
+
     const payloadFilter = EventFilterSchema.safeParse(dispatcher.payloadFilter ?? {});
 
     const contextFilter = EventFilterSchema.safeParse(dispatcher.contextFilter ?? {});

apps/webapp/app/services/events/invokeDispatcher.server.ts

@@ -3,21 +3,8 @@ import type { PrismaClientOrTransaction } from "~/db.server";
 import { prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { CreateRunService } from "~/services/runs/createRun.server";
-
-const JobVersionDispatchableSchema = z.object({
-  type: z.literal("JOB_VERSION"),
-  id: z.string(),
-});
-
-const DynamicTriggerDispatchableSchema = z.object({
-  type: z.literal("DYNAMIC_TRIGGER"),
-  id: z.string(),
-});
-
-const DispatchableSchema = z.discriminatedUnion("type", [
-  JobVersionDispatchableSchema,
-  DynamicTriggerDispatchableSchema,
-]);
+import { InvokeEphemeralDispatcherService } from "../dispatchers/invokeEphemeralEventDispatcher.server";
+import { DispatchableSchema } from "~/models/eventDispatcher.server";
 
 export class InvokeDispatcherService {
   #prismaClient: PrismaClientOrTransaction;
@@ -142,6 +129,11 @@ export class InvokeDispatcherService {
           });
         }
 
+        break;
+      }
+      case "EPHEMERAL": {
+        await InvokeEphemeralDispatcherService.enqueue(eventDispatcher.id, eventRecord.id);
+
         break;
       }
     }

apps/webapp/app/services/events/sqsEventConsumer.ts

@@ -0,0 +1,151 @@
+import { Consumer } from "sqs-consumer";
+import { PrismaClientOrTransaction, prisma } from "~/db.server";
+import { logger, trace } from "../logger.server";
+import { Message, SQSClient } from "@aws-sdk/client-sqs";
+import { authenticateApiKey } from "../apiAuth.server";
+import { SendEventBodySchema } from "@trigger.dev/core";
+import { z } from "zod";
+import { fromZodError } from "zod-validation-error";
+import { IngestSendEvent } from "./ingestSendEvent.server";
+import { env } from "~/env.server";
+import { singleton } from "~/utils/singleton";
+
+type SqsEventConsumerOptions = {
+  queueUrl: string;
+  /** This cannot be higher than the AWS limit of 10. */
+  batchSize: number;
+  region: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+};
+
+const messageSchema = SendEventBodySchema.extend({
+  apiKey: z.string(),
+});
+
+export class SqsEventConsumer {
+  readonly #ingestEventService: IngestSendEvent;
+  readonly #consumer: Consumer;
+
+  constructor(
+    readonly prismaClient: PrismaClientOrTransaction = prisma,
+    options: SqsEventConsumerOptions
+  ) {
+    this.#ingestEventService = new IngestSendEvent();
+
+    logger.debug("SqsEventConsumer starting", {
+      queueUrl: options.queueUrl,
+      region: options.region,
+    });
+
+    this.#consumer = Consumer.create({
+      queueUrl: options.queueUrl,
+      batchSize: options.batchSize,
+      sqs: new SQSClient({
+        region: options.region,
+        credentials: {
+          accessKeyId: options.accessKeyId,
+          secretAccessKey: options.secretAccessKey,
+        },
+      }),
+      handleMessage: async (message) => {
+        await trace({ sqsMessage: message }, async () => await this.#processEvent(message));
+      },
+    });
+
+    this.#consumer.on("error", (err, message) => {
+      logger.error("SqsEventConsumer error", { error: err.message, sqsMessage: message });
+      //todo what do we want to do here?
+    });
+
+    this.#consumer.on("processing_error", (err, message) => {
+      logger.error("SqsEventConsumer processing_error", {
+        error: err.message,
+        sqsMessage: message,
+      });
+      //todo what do we want to do here?
+    });
+
+    this.#consumer.on("timeout_error", (err, message) => {
+      logger.error("SqsEventConsumer timeout_error", { error: err.message, sqsMessage: message });
+      //todo what do we want to do here?
+    });
+
+    //Stop the consumer if the process is terminated
+    process.on("SIGTERM", () => {
+      this.stop();
+    });
+
+    this.#consumer.start();
+  }
+
+  public stop() {
+    logger.debug("SqsEventConsumer stopping");
+    this.#consumer.stop({ abort: true });
+  }
+
+  async #processEvent(message: Message) {
+    logger.debug("SqsEventConsumer processing event");
+
+    //parse the body
+    if (!message.Body) {
+      logger.error("SqsEventConsumer message has no body");
+      return;
+    }
+
+    const body = messageSchema.safeParse(JSON.parse(message.Body));
+    if (!body.success) {
+      logger.error("SqsEventConsumer message body is invalid", {
+        error: fromZodError(body.error).message,
+      });
+      return;
+    }
+
+    //authenticate API Key
+    const authenticationResult = await authenticateApiKey(body.data.apiKey);
+    if (!authenticationResult) {
+      logger.warn("SqsEventConsumer message has invalid API key");
+      return;
+    }
+
+    const authenticatedEnv = authenticationResult.environment;
+
+    logger.info("sqs_event", { event: body.data.event, options: body.data.options });
+
+    const event = await this.#ingestEventService.call(
+      authenticatedEnv,
+      body.data.event,
+      body.data.options
+    );
+
+    if (!event) {
+      logger.error("SqsEventConsumer failed to create event");
+      return;
+    }
+
+    logger.debug("SqsEventConsumer processed event", { event });
+  }
+}
+
+export function getSharedSqsEventConsumer() {
+  if (
+    env.AWS_SQS_QUEUE_URL &&
+    env.AWS_SQS_REGION &&
+    env.AWS_SQS_ACCESS_KEY_ID &&
+    env.AWS_SQS_SECRET_ACCESS_KEY
+  ) {
+    const consumer = new SqsEventConsumer(undefined, {
+      queueUrl: env.AWS_SQS_QUEUE_URL,
+      batchSize: env.AWS_SQS_BATCH_SIZE,
+      region: env.AWS_SQS_REGION,
+      accessKeyId: env.AWS_SQS_ACCESS_KEY_ID,
+      secretAccessKey: env.AWS_SQS_SECRET_ACCESS_KEY,
+    });
+
+    return consumer;
+  }
+
+  console.log(
+    "The SqsEventConsumer is disabled because AWS credentials are missing. This is OK as this is an optional feature."
+  );
+}

apps/webapp/app/services/executions/createExecutionEvent.server.ts

@@ -0,0 +1,54 @@
+import { PrismaClientOrTransaction, prisma } from "~/db.server";
+import { logger } from "../logger.server";
+
+export type CreateExecutionEventInput = {
+  organizationId: string;
+  projectId: string;
+  environmentId: string;
+  jobId: string;
+  runId: string;
+  eventTime: Date;
+  eventType: "start" | "finish";
+  drift?: number;
+};
+
+export class CreateExecutionEventService {
+  constructor(private prismaClient: PrismaClientOrTransaction = prisma) {}
+
+  public async call(input: CreateExecutionEventInput) {
+    await this.prismaClient.$executeRaw`
+      INSERT INTO "triggerdotdev_events"."run_executions" (
+        "organization_id",
+        "project_id",
+        "environment_id",
+        "job_id",
+        "run_id",
+        "event_time",
+        "event_type",
+        "drift_amount_in_ms"
+      ) VALUES (
+        ${input.organizationId},
+        ${input.projectId},
+        ${input.environmentId},
+        ${input.jobId},
+        ${input.runId},
+        ${input.eventTime},
+        ${input.eventType === "start" ? 1 : -1},
+        ${input.drift}
+      )
+    `;
+  }
+}
+
+export async function createExecutionEvent(
+  input: CreateExecutionEventInput,
+  options?: { prismaClient?: PrismaClientOrTransaction }
+) {
+  const service = new CreateExecutionEventService(options?.prismaClient);
+
+  try {
+    return await service.call(input);
+  } catch (error) {
+    logger.error("Error creating execution event", { error });
+  }
+}

apps/webapp/app/services/externalApis/apis.server.ts

@@ -0,0 +1,804 @@
+export type Api = {
+  identifier: string;
+  name: string;
+  examples?: ApiExample[];
+};
+
+export type ApiExample = {
+  title: string;
+  version: string;
+  codeUrl: string;
+  slug: string;
+};
+
+export const apisList = [
+  {
+    identifier: "airtable",
+    name: "Airtable",
+    examples: [
+      {
+        title: "Update Airtable when a new subscription is added to Stripe.",
+        slug: "stripe-sub-update-airtable",
+        version: "1.0.0",
+
+        codeUrl:
+          "https://raw.githubusercontent.com/triggerdotdev/jobs-showcase/main/src/stripeNewSubscriptionUpdateAirtable.ts",
+      },
+      {
+        title: "Add a new record to Airtable when a Typeform response is submitted.",
+        version: "1.0.0",
+        slug: "new-airtable-record-from-typeform",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/typeformNewSubmissionUpdateAirtable.ts",
+      },
+      {
+        title: "Update Airtable database when there is a sale in Stripe.",
+        version: "1.0.0",
+        slug: "update-airtable-when-stripe-account-updated",
+        codeUrl:
+          "https://raw.githubusercontent.com/triggerdotdev/jobs-showcase/main/src/syncStripeWithAirtable.ts",
+      },
+    ],
+  },
+  {
+    identifier: "algolia",
+    name: "Algolia",
+  },
+  {
+    identifier: "anthropic",
+    name: "Anthropic",
+  },
+  {
+    identifier: "appsmith",
+    name: "Appsmith",
+  },
+  {
+    identifier: "appwrite",
+    name: "Appwrite",
+  },
+  {
+    identifier: "asana",
+    name: "Asana",
+    examples: [
+      {
+        title: "Get user details from Asana",
+        slug: "get-user-details",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/asana.ts",
+      },
+    ],
+  },
+  {
+    identifier: "atlassian",
+    name: "Atlassian",
+  },
+  {
+    identifier: "aws",
+    name: "AWS",
+    examples: [
+      {
+        title: "Trigger an AWS Lambda function with a defined payload and log the results.",
+        slug: "get-user-details",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/aws.ts",
+      },
+    ],
+  },
+  {
+    identifier: "brex",
+    name: "Brex",
+    examples: [
+      {
+        title: "Create a new title in a Brex account.",
+        slug: "create-new-brex-title",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/brex.ts",
+      },
+    ],
+  },
+  {
+    identifier: "caldotcom",
+    name: "Cal.com",
+    examples: [
+      {
+        title: "Send a Slack message when meetings are booked or cancelled.",
+        slug: "cal-slack-meeting-alert",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/cal-http-endpoint.ts",
+      },
+      {
+        title: "Find all Cal.com bookings for a user.",
+        slug: "cal-find-bookings",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/cal.ts",
+      },
+    ],
+  },
+  {
+    identifier: "clerk",
+    name: "Clerk",
+  },
+  {
+    identifier: "clickup",
+    name: "ClickUp",
+  },
+  {
+    identifier: "coda",
+    name: "Coda",
+  },
+  {
+    identifier: "crowddotdev",
+    name: "Crowd.dev",
+  },
+  {
+    identifier: "deepl",
+    name: "DeepL",
+    examples: [
+      {
+        title: "Translate some text with DeepL.",
+        slug: "translate-text-with-deepl",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/deepl.ts",
+      },
+    ],
+  },
+  {
+    identifier: "discord",
+    name: "Discord",
+    examples: [
+      {
+        title: "Create a Discord bot and send a message to a channel.",
+        slug: "discord-bot-send-message",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/discord.ts",
+      },
+    ],
+  },
+  {
+    identifier: "documenso",
+    name: "Documenso",
+  },
+  {
+    identifier: "dropbox",
+    name: "Dropbox",
+  },
+  {
+    identifier: "facebook",
+    name: "Facebook",
+  },
+  {
+    identifier: "fastify",
+    name: "Fastify",
+  },
+  {
+    identifier: "flickr",
+    name: "Flickr",
+  },
+  {
+    identifier: "github",
+    name: "GitHub",
+    examples: [
+      {
+        title: "Send a message to a Slack channel when a repo is starred.",
+        slug: "github-star-to-slack",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/gitHubNewStarToSlack.ts",
+      },
+      {
+        title: "Create a Linear issue when a pull request is opened on a GitHub repo.",
+        slug: "linear-ticket-on-github-pr",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/linearCreateIssueOnPR.ts",
+      },
+      {
+        title:
+          "Send a reminder message to a Slack channel if a GitHub issue is left open for 24 hours.",
+        slug: "github-issue-reminder",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/gitHubIssueReminder.ts",
+      },
+
+      {
+        title: "Add a custom label to a GitHub issue.",
+        slug: "github-custom-label",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/gitHubNewIssueOpened.ts",
+      },
+    ],
+  },
+  {
+    identifier: "giphy",
+    name: "Giphy",
+  },
+  {
+    identifier: "gmail",
+    name: "Gmail",
+    examples: [
+      {
+        title: "Send an email using Gmail.",
+        slug: "send-email-with-gmail",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/gmail.ts",
+      },
+    ],
+  },
+  {
+    identifier: "googlecalendar",
+    name: "Google Calendar",
+    examples: [
+      {
+        title: "Create a new Google Calendar event",
+        slug: "create-google-calendar-event",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/google-calendar.ts",
+      },
+    ],
+  },
+  {
+    identifier: "googledocs",
+    name: "Google Docs",
+  },
+  {
+    identifier: "googledrive",
+    name: "Google Drive",
+    examples: [
+      {
+        title: "Update a filename in Google Drive.",
+        slug: "update-google-drive-filename",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/google-drive.ts",
+      },
+    ],
+  },
+  {
+    identifier: "googlemaps",
+    name: "Google Maps",
+    examples: [
+      {
+        title: "Make a geocode request with Google Maps.",
+        slug: "google-maps-geocode",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/google-maps.ts",
+      },
+    ],
+  },
+  {
+    identifier: "googlesheets",
+    name: "Google Sheets",
+    examples: [
+      {
+        title: "Insert data into a row in Google Sheets.",
+        slug: "insert-data-into-google-sheets",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/google-sheets.ts",
+      },
+    ],
+  },
+  {
+    identifier: "hubspot",
+    name: "HubSpot",
+    examples: [
+      {
+        title: "Create a contact in HubSpot.",
+        slug: "create-contact-in-hubspot",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/hubspot.ts",
+      },
+    ],
+  },
+  {
+    identifier: "huggingface",
+    name: "Hugging Face",
+    examples: [
+      {
+        title: "Text classification with Hugging Face.",
+        slug: "text-classification-with-hugging-face",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/hugging-face.ts",
+      },
+    ],
+  },
+  {
+    identifier: "infisical",
+    name: "Infisical",
+  },
+  {
+    identifier: "instagram",
+    name: "Instagram",
+    examples: [
+      {
+        title: "Post an image to Instagram",
+        slug: "post-image-to-instagram",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/instagram.ts",
+      },
+    ],
+  },
+  {
+    identifier: "instabug",
+    name: "Instabug",
+  },
+  {
+    identifier: "keep",
+    name: "Keep",
+  },
+  {
+    identifier: "lemonsqueezy",
+    name: "Lemon Squeezy",
+    examples: [
+      {
+        title: "Get store information from Lemon Squeezy.",
+        slug: "get-store-information",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/lemon-squeezy.ts",
+      },
+    ],
+  },
+  {
+    identifier: "linkedin",
+    name: "LinkedIn",
+  },
+  {
+    identifier: "linear",
+    name: "Linear",
+    examples: [
+      {
+        title: "Post Linear issues to Slack every weekday at 9am using Cron.",
+        slug: "daily-linear-issues-slack-alert",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/linearIssuesDailySlackAlert.ts",
+      },
+      {
+        title: "Create a Linear issue when a pull request is opened on a GitHub repo.",
+        slug: "linear-ticket-on-pr",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/linearCreateIssueOnPR.ts",
+      },
+      {
+        title: "Automatically comment and like any new Linear issues.",
+        slug: "automatically-comment-and-like-linear-issues",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/linearNewIssueReply.ts",
+      },
+    ],
+  },
+  {
+    identifier: "loops",
+    name: "Loops",
+    examples: [
+      {
+        title: "Create a new contact in Loops.",
+        slug: "create-new-contact-in-loops",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/loops.ts",
+      },
+    ],
+  },
+  {
+    identifier: "lotus",
+    name: "Lotus",
+  },
+  {
+    identifier: "mailchimp",
+    name: "Mailchimp",
+  },
+  {
+    identifier: "mailgun",
+    name: "Mailgun",
+    examples: [
+      {
+        title: "Send an email with Mailgun.",
+        slug: "send-email-with-mailgun",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/mailgun.ts",
+      },
+    ],
+  },
+  {
+    identifier: "microsoftazure",
+    name: "Microsoft Azure",
+  },
+  {
+    identifier: "monday",
+    name: "Monday",
+  },
+  {
+    identifier: "mux",
+    name: "Mux",
+  },
+  {
+    identifier: "notion",
+    name: "Notion",
+    examples: [
+      {
+        title: "Retrieve a Notion page by ID.",
+        slug: "retrieve-notion-page",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/notion.ts",
+      },
+    ],
+  },
+  {
+    identifier: "novu",
+    name: "Novu",
+    examples: [
+      {
+        title: "Create a new subscriber in Novu",
+        slug: "create-new-subscriber-in-novu",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/novu.ts",
+      },
+    ],
+  },
+  {
+    identifier: "openai",
+    name: "OpenAI",
+    examples: [
+      {
+        title: "Summarize GitHub commits using OpenAI and then post them to Slack.",
+        slug: "openai-summarize-github-commits",
+        version: "1.0.0",
+        codeUrl:
+          "https://raw.githubusercontent.com/triggerdotdev/jobs-showcase/main/src/summarizeGitHubCommits.ts",
+      },
+      {
+        title: "Generate a random joke using OpenAI.",
+        slug: "openai-generate-random-joke",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/openAITellMeAJoke.ts",
+      },
+      {
+        title: "Generate an image from a prompt using OpenAI.",
+        slug: "openai-generate-image",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/openAIGenerateImage.ts",
+      },
+    ],
+  },
+  {
+    identifier: "pagerduty",
+    name: "PagerDuty",
+  },
+  {
+    identifier: "plain",
+    name: "Plain",
+    examples: [
+      {
+        title: "Update or create customer information based on an identifier.",
+        slug: "plain-update-customer-information",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/plainUpdateCustomer.ts",
+      },
+    ],
+  },
+  {
+    identifier: "posthog",
+    name: "Posthog",
+  },
+  {
+    identifier: "raycast",
+    name: "Raycast",
+  },
+  {
+    identifier: "reddit",
+    name: "Reddit",
+  },
+  {
+    identifier: "replicate",
+    name: "Replicate",
+    examples: [
+      {
+        title: "Generate a cinematic image with Replicate.",
+        slug: "generate-cinematic-image",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/replicateCinematicPrompt.ts",
+      },
+    ],
+  },
+  {
+    identifier: "resend",
+    name: "Resend",
+    examples: [
+      {
+        title: "Send a drip email campaign over 30 days, triggered by an event.",
+        slug: "resend-send-drip-campaign",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/resendDripCampaign.tsx",
+      },
+      {
+        title: "Send an email built using React with Resend.",
+        slug: "send-react-email",
+        version: "1.0.0",
+        codeUrl:
+          "https://raw.githubusercontent.com/triggerdotdev/jobs-showcase/main/src/resendSendReactEmail.tsx",
+      },
+      {
+        title: "Send a basic email with Resend.",
+        slug: "resend-send-basic-email",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/resendSendBasicEmail.ts",
+      },
+    ],
+  },
+  {
+    identifier: "salesforce",
+    name: "Salesforce",
+    examples: [
+      {
+        title: "Create a new contact in Salesforce.",
+        slug: "salesforce-create-contact",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/salesforce.ts",
+      },
+    ],
+  },
+  {
+    identifier: "segment",
+    name: "Segment",
+    examples: [
+      {
+        title: "Get source information from Segment.",
+        slug: "segment-get-source-information",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/segment.ts",
+      },
+    ],
+  },
+  {
+    identifier: "sendgrid",
+    name: "SendGrid",
+    examples: [
+      {
+        title: "Send an activity summary email to users at 4pm every Friday.",
+        slug: "sendgrid-send-activity-summary",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/weeklyUserActivitySummary.ts",
+      },
+      {
+        title: "SendGrid send basic email.",
+        slug: "sendgrid-send-basic-email",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/sendGridSendBasicEmail.ts",
+      },
+    ],
+  },
+  {
+    identifier: "shopify",
+    name: "Shopify",
+    examples: [
+      {
+        title: "Update a product variant price in Shopify.",
+        slug: "shopify-update-product-variant-price",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/shopify.ts",
+      },
+    ],
+  },
+  {
+    identifier: "slack",
+    name: "Slack",
+    examples: [
+      {
+        title: "Posts Linear issues to Slack every weekday at 9am using Cron.",
+        slug: "slack-daily-linear-issues",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/linearIssuesDailySlackAlert.ts",
+      },
+      {
+        title: "Summarize GitHub commits using OpenAI and then post them to Slack.",
+        slug: "slack-openai-summarize-github-commits",
+        version: "1.0.0",
+        codeUrl:
+          "https://raw.githubusercontent.com/triggerdotdev/jobs-showcase/main/src/summarizeGitHubCommits.ts",
+      },
+      {
+        title: "Send an activity summary email, and post it to Slack at 4pm every Friday.",
+        slug: "slack-sendgrid-send-activity-summary",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/weeklyUserActivitySummary.ts",
+      },
+      {
+        title: "Send a message to a Slack channel when a GitHub repo is starred.",
+        slug: "slack-post-github",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/gitHubNewStarToSlack.ts",
+      },
+      {
+        title:
+          "Send a reminder message to a Slack channel if a GitHub issue is left open for 24 hours.",
+        slug: "slack-github-issue-reminder",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/gitHubIssueReminder.ts",
+      },
+    ],
+  },
+  {
+    identifier: "snyk",
+    name: "Snyk",
+    examples: [
+      {
+        title: "Get user details from Snyk.",
+        slug: "snyk-get-user-details",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/snyk.ts",
+      },
+    ],
+  },
+  {
+    identifier: "spotify",
+    name: "Spotify",
+  },
+  {
+    identifier: "stabilityai",
+    name: "Stability AI",
+    examples: [
+      {
+        title: "Generate an image with Stability AI.",
+        slug: "stabilityai-generate-image",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/stability.ts",
+      },
+    ],
+  },
+  {
+    identifier: "stripe",
+    name: "Stripe",
+    examples: [
+      {
+        title: "Update Supabase every time a Stripe account is updated.",
+        slug: "stripe-supabase-update",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/supabaseStripeUpdateDatabase.ts",
+      },
+      {
+        title: "Update Airtable when a new subscription is added to Stripe.",
+        slug: "stripe-sub-update-airtable",
+        version: "1.0.0",
+        codeUrl:
+          "https://raw.githubusercontent.com/triggerdotdev/jobs-showcase/main/src/stripeNewSubscriptionUpdateAirtable.ts",
+      },
+      {
+        title: "Update Airtable database when there is a sale in Stripe.",
+        version: "1.0.0",
+        slug: "update-airtable-when-stripe-account-updated",
+        codeUrl:
+          "https://raw.githubusercontent.com/triggerdotdev/jobs-showcase/main/src/syncStripeWithAirtable.ts",
+      },
+    ],
+  },
+  {
+    identifier: "supabase",
+    name: "Supabase",
+    examples: [
+      {
+        title: "Update Supabase every time a Stripe account is updated.",
+        slug: "stripe-supabase-update",
+        version: "1.0.0",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/supabaseStripeUpdateDatabase.ts",
+      },
+    ],
+  },
+  {
+    identifier: "svix",
+    name: "Svix",
+    examples: [
+      {
+        title: "Create an application in Svix",
+        slug: "svix-create-application",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/svix.ts",
+      },
+    ],
+  },
+  {
+    identifier: "todoist",
+    name: "Todoist",
+    examples: [
+      {
+        title: "Add a new project in Todoist.",
+        slug: "todoist-add-new-project",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/todoist.ts",
+      },
+    ],
+  },
+  {
+    identifier: "trello",
+    name: "Trello",
+  },
+
+  {
+    identifier: "twilio",
+    name: "Twilio",
+    examples: [
+      {
+        title: "Send an SMS or WhatsApp message with Twilio",
+        slug: "twilio-send-sms-or-whatsapp-message",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/twilio.ts",
+      },
+    ],
+  },
+  {
+    identifier: "typeform",
+    name: "Typeform",
+    examples: [
+      {
+        title: "Add a new record to Airtable when a Typeform response is submitted.",
+        version: "1.0.0",
+        slug: "new-airtable-record-from-typeform",
+        codeUrl:
+          "https://github.com/triggerdotdev/jobs-showcase/raw/main/src/typeformNewSubmissionUpdateAirtable.ts",
+      },
+    ],
+  },
+
+  {
+    identifier: "whatsapp",
+    name: "WhatsApp",
+    examples: [
+      {
+        title: "Send a message to a WhatsApp number",
+        slug: "whatapp-send-message",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/whatsapp.ts",
+      },
+    ],
+  },
+  {
+    identifier: "x",
+    name: "X (Twitter)",
+    examples: [
+      {
+        title: "Post a post to an X (Twitter) account",
+        slug: "post-to-x",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/x.ts",
+      },
+    ],
+  },
+  {
+    identifier: "youtube",
+    name: "YouTube",
+    examples: [
+      {
+        title: "Search for a YouTube video",
+        slug: "youtube-search-video",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/youtube.ts",
+      },
+    ],
+  },
+  {
+    identifier: "zbd",
+    name: "ZBD",
+    examples: [
+      {
+        title: "Send Satoshis to a ZBD account.",
+        slug: "zbd-send-satoshis",
+        version: "1.0.0",
+        codeUrl: "https://github.com/triggerdotdev/api-reference/raw/main/src/zbd.ts",
+      },
+    ],
+  },
+];

apps/webapp/app/services/externalApis/apis.ts

@@ -1,223 +0,0 @@
-export type Api = {
-  identifier: string;
-  name: string;
-};
-
-export const apisList = [
-  {
-    identifier: "airtable",
-    name: "Airtable",
-  },
-  {
-    identifier: "algolia",
-    name: "Algolia",
-  },
-  {
-    identifier: "anthropic",
-    name: "Anthropic",
-  },
-  {
-    identifier: "appsmith",
-    name: "Appsmith",
-  },
-  {
-    identifier: "asana",
-    name: "Asana",
-  },
-  {
-    identifier: "aws",
-    name: "AWS",
-  },
-  {
-    identifier: "caldotcom",
-    name: "Cal.com",
-  },
-  {
-    identifier: "clickup",
-    name: "ClickUp",
-  },
-  {
-    identifier: "coda",
-    name: "Coda",
-  },
-  {
-    identifier: "discord",
-    name: "Discord",
-  },
-  {
-    identifier: "dropbox",
-    name: "Dropbox",
-  },
-  {
-    identifier: "facebook",
-    name: "Facebook",
-  },
-  {
-    identifier: "flickr",
-    name: "Flickr",
-  },
-  {
-    identifier: "github",
-    name: "GitHub",
-  },
-  {
-    identifier: "giphy",
-    name: "Giphy",
-  },
-  {
-    identifier: "gmail",
-    name: "Gmail",
-  },
-  {
-    identifier: "googlecalendar",
-    name: "Google Calendar",
-  },
-  {
-    identifier: "googlecloudplatform",
-    name: "Google Cloud Platform",
-  },
-  {
-    identifier: "googledocs",
-    name: "Google Docs",
-  },
-  {
-    identifier: "googledrive",
-    name: "Google Drive",
-  },
-  {
-    identifier: "googlemaps",
-    name: "Google Maps",
-  },
-  {
-    identifier: "hubspot",
-    name: "HubSpot",
-  },
-  {
-    identifier: "instagram",
-    name: "Instagram",
-  },
-  {
-    identifier: "infisical",
-    name: "Infisical",
-  },
-  {
-    identifier: "linkedin",
-    name: "LinkedIn",
-  },
-  {
-    identifier: "linear",
-    name: "Linear",
-  },
-  {
-    identifier: "loops",
-    name: "Loops",
-  },
-  {
-    identifier: "mailchimp",
-    name: "Mailchimp",
-  },
-  {
-    identifier: "mailgun",
-    name: "Mailgun",
-  },
-  {
-    identifier: "microsoftazure",
-    name: "Microsoft Azure",
-  },
-  {
-    identifier: "mux",
-    name: "Mux",
-  },
-  {
-    identifier: "notion",
-    name: "Notion",
-  },
-  {
-    identifier: "openai",
-    name: "OpenAI",
-  },
-  {
-    identifier: "plain",
-    name: "Plain",
-  },
-  {
-    identifier: "posthog",
-    name: "Posthog",
-  },
-  {
-    identifier: "reddit",
-    name: "Reddit",
-  },
-  {
-    identifier: "replicate",
-    name: "Replicate",
-  },
-  {
-    identifier: "resend",
-    name: "Resend",
-  },
-  {
-    identifier: "salesforce",
-    name: "Salesforce",
-  },
-  {
-    identifier: "segment",
-    name: "Segment",
-  },
-  {
-    identifier: "sendgrid",
-    name: "SendGrid",
-  },
-  {
-    identifier: "shopify",
-    name: "Shopify",
-  },
-  {
-    identifier: "slack",
-    name: "Slack",
-  },
-  {
-    identifier: "spotify",
-    name: "Spotify",
-  },
-  {
-    identifier: "stabilityai",
-    name: "Stability AI",
-  },
-  {
-    identifier: "stripe",
-    name: "Stripe",
-  },
-  {
-    identifier: "supabase",
-    name: "Supabase",
-  },
-  {
-    identifier: "svix",
-    name: "Svix",
-  },
-  {
-    identifier: "trello",
-    name: "Trello",
-  },
-  {
-    identifier: "twilio",
-    name: "Twilio",
-  },
-  {
-    identifier: "twitter",
-    name: "Twitter",
-  },
-  {
-    identifier: "typeform",
-    name: "Typeform",
-  },
-  {
-    identifier: "whatsapp",
-    name: "WhatsApp",
-  },
-  {
-    identifier: "youtube",
-    name: "YouTube",
-  },
-];

apps/webapp/app/services/httpendpoint/HandleHttpEndpointService.ts

@@ -80,7 +80,7 @@ export class HandleHttpEndpointService {
       });
 
     if (!httpEndpointEnvironment) {
-      logger.error("Could not find http endpoint environment", {
+      logger.debug("Could not find http endpoint environment", {
         httpEndpointId: httpEndpoint.id,
         environmentId: environment.id,
       });

apps/webapp/app/services/logger.server.ts

@@ -1,26 +1,32 @@
 import type { LogLevel } from "@trigger.dev/core";
 import { Logger } from "@trigger.dev/core";
 import { sensitiveDataReplacer } from "./sensitiveDataReplacer";
-import { singleton } from "~/utils/singleton";
+import { AsyncLocalStorage } from "async_hooks";
 
-export const logger = singleton(
-  "logger",
-  () =>
-    new Logger(
-      "webapp",
-      (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
-      ["examples", "output", "connectionString", "payload"],
-      sensitiveDataReplacer
-    )
+const currentFieldsStore = new AsyncLocalStorage<Record<string, unknown>>();
+
+export function trace<T>(fields: Record<string, unknown>, fn: () => T): T {
+  return currentFieldsStore.run(fields, fn);
+}
+
+export const logger = new Logger(
+  "webapp",
+  (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+  ["examples", "output", "connectionString", "payload"],
+  sensitiveDataReplacer,
+  () => {
+    const fields = currentFieldsStore.getStore();
+    return fields ? { ...fields } : {};
+  }
 );
 
-export const workerLogger = singleton(
-  "worker-logger",
-  () =>
-    new Logger(
-      "worker",
-      (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
-      ["examples", "output", "connectionString"],
-      sensitiveDataReplacer
-    )
+export const workerLogger = new Logger(
+  "worker",
+  (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+  ["examples", "output", "connectionString"],
+  sensitiveDataReplacer,
+  () => {
+    const fields = currentFieldsStore.getStore();
+    return fields ? { ...fields } : {};
+  }
 );

apps/webapp/app/services/runs/deliverRunSubscription.server.ts

@@ -1,6 +1,7 @@
 import { RunNotification } from "@trigger.dev/core";
 import { subtle } from "node:crypto";
 import { PrismaClient, prisma } from "~/db.server";
+import { EndpointApi } from "../endpointApi.server";
 
 // Infer the type of the #findSubscription method
 type FoundSubscription = NonNullable<
@@ -87,6 +88,34 @@ export class DeliverRunSubscriptionService {
           );
         }
 
+        return true;
+      }
+      case "ENDPOINT": {
+        const endpointId = subscription.recipient;
+
+        if (endpointId !== subscription.run.endpointId) {
+          return true;
+        }
+
+        const client = new EndpointApi(
+          subscription.run.environment.apiKey,
+          subscription.run.endpoint.url
+        );
+
+        const response = await client.deliverRunNotification(payload);
+
+        if (!response) {
+          throw new Error(
+            `Failed to deliver endpoint notification to ${subscription.run.endpoint.url}`
+          );
+        }
+
+        if (!response.ok) {
+          throw new Error(
+            `Failed to deliver endpoint notification to ${subscription.run.endpoint.url}: [${response.status}] ${response.statusText}`
+          );
+        }
+
         return true;
       }
     }
@@ -107,6 +136,19 @@ export class DeliverRunSubscriptionService {
             organization: true,
             project: true,
             event: true,
+            endpoint: true,
+            tasks: {
+              where: {
+                status: "ERRORED",
+              },
+              take: 1,
+              orderBy: {
+                startedAt: "desc",
+              },
+              include: {
+                attempts: true,
+              },
+            },
           },
         },
       },
@@ -114,7 +156,20 @@ export class DeliverRunSubscriptionService {
   }
 
   #getPayload(run: FoundRun): RunNotification<any> {
-    const { id, job, version, statuses, environment, organization, project, event } = run;
+    const { id, job, version, statuses, environment, organization, project, event, tasks } = run;
+
+    const task = tasks[0]
+      ? {
+          id: tasks[0].idempotencyKey,
+          cacheKey: tasks[0].displayKey,
+          status: tasks[0].status,
+          name: tasks[0].name,
+          icon: tasks[0].icon,
+          startedAt: tasks[0].startedAt,
+          error: tasks[0].output,
+          params: tasks[0].params,
+        }
+      : undefined;
 
     const payload = {
       id,
@@ -123,10 +178,11 @@ export class DeliverRunSubscriptionService {
       startedAt: run.startedAt,
       updatedAt: run.updatedAt,
       completedAt: run.completedAt,
+      isTest: run.isTest,
       executionDurationInMs: run.executionDuration,
       executionCount: run.executionCount,
       job: {
-        id: job.id,
+        id: job.slug,
         version: version.version,
       },
       statuses: statuses.map((status) => ({
@@ -155,8 +211,14 @@ export class DeliverRunSubscriptionService {
         id: event.id,
         context: event.context,
         timestamp: event.timestamp,
+        payload: event.payload,
       },
-      ...(run.status === "SUCCESS" ? { output: run.output } : { error: run.output }),
+      ...(run.status === "SUCCESS"
+        ? { output: run.output }
+        : {
+            error: run.output,
+            task,
+          }),
     };
 
     return payload as RunNotification<any>;

apps/webapp/app/services/runs/performRunExecutionV3.server.ts

@@ -2,7 +2,6 @@ import {
   ApiEventLog,
   AutoYieldMetadata,
   ConnectionAuth,
-  EndpointHeadersSchema,
   RunJobAutoYieldWithCompletedTaskExecutionError,
   RunJobBody,
   RunJobError,
@@ -34,10 +33,11 @@ import { CompleteRunTaskService } from "~/routes/api.v1.runs.$runId.tasks.$id.co
 import { formatError } from "~/utils/formatErrors.server";
 import { safeJsonZodParse } from "~/utils/json";
 import { EndpointApi } from "../endpointApi.server";
+import { createExecutionEvent } from "../executions/createExecutionEvent.server";
 import { logger } from "../logger.server";
-import { forceYieldCoordinator } from "./forceYieldCoordinator.server";
-import { workerQueue } from "../worker.server";
 import { ResumeTaskService } from "../tasks/resumeTask.server";
+import { workerQueue } from "../worker.server";
+import { forceYieldCoordinator } from "./forceYieldCoordinator.server";
 
 type FoundRun = NonNullable<Awaited<ReturnType<typeof findRun>>>;
 type FoundTask = FoundRun["tasks"][number];
@@ -67,7 +67,7 @@ export class PerformRunExecutionV3Service {
     this.#prismaClient = prismaClient;
   }
 
-  public async call(input: PerformRunExecutionV3Input) {
+  public async call(input: PerformRunExecutionV3Input, driftInMs: number = 0) {
     const run = await findRun(this.#prismaClient, input.id);
 
     if (!run) {
@@ -80,7 +80,7 @@ export class PerformRunExecutionV3Service {
         break;
       }
       case "EXECUTE_JOB": {
-        await this.#executeJob(run, input);
+        await this.#executeJob(run, input, driftInMs);
         break;
       }
     }
@@ -177,7 +177,7 @@ export class PerformRunExecutionV3Service {
       });
     }
   }
-  async #executeJob(run: FoundRun, input: PerformRunExecutionV3Input) {
+  async #executeJob(run: FoundRun, input: PerformRunExecutionV3Input, driftInMs: number = 0) {
     try {
       const { isRetry, resumeTaskId } = input;
 
@@ -275,9 +275,30 @@ export class PerformRunExecutionV3Service {
 
       forceYieldCoordinator.registerRun(run.id);
 
-      const { response, parser, errorParser, durationInMs } = await client.executeJobRequest(
-        executionBody
-      );
+      await createExecutionEvent({
+        eventType: "start",
+        eventTime: new Date(),
+        drift: driftInMs,
+        organizationId: run.organizationId,
+        environmentId: run.environmentId,
+        projectId: run.projectId,
+        jobId: run.jobId,
+        runId: run.id,
+      });
+
+      const { response, parser, errorParser, headersParser, durationInMs } =
+        await client.executeJobRequest(executionBody);
+
+      await createExecutionEvent({
+        eventType: "finish",
+        eventTime: new Date(),
+        drift: 0,
+        organizationId: run.organizationId,
+        environmentId: run.environmentId,
+        projectId: run.projectId,
+        jobId: run.jobId,
+        runId: run.id,
+      });
 
       forceYieldCoordinator.deregisterRun(run.id);
 
@@ -289,7 +310,7 @@ export class PerformRunExecutionV3Service {
 
       // Update the endpoint version if it has changed
       const rawHeaders = Object.fromEntries(response.headers.entries());
-      const headers = EndpointHeadersSchema.safeParse(rawHeaders);
+      const headers = headersParser.safeParse(rawHeaders);
 
       if (
         headers.success &&
@@ -306,6 +327,58 @@ export class PerformRunExecutionV3Service {
         });
       }
 
+      if (headers.success && headers.data["x-trigger-run-metadata"] && !run.internal) {
+        logger.debug("Endpoint responded with run metadata", {
+          metadata: headers.data["x-trigger-run-metadata"],
+        });
+
+        if (
+          headers.data["x-trigger-run-metadata"].successSubscription &&
+          !run.subscriptions.some((s) => s.event === "SUCCESS")
+        ) {
+          await this.#prismaClient.jobRunSubscription.upsert({
+            where: {
+              runId_recipient_event: {
+                runId: run.id,
+                recipient: run.endpoint.id,
+                event: "SUCCESS",
+              },
+            },
+            create: {
+              runId: run.id,
+              recipient: run.endpoint.id,
+              recipientMethod: "ENDPOINT",
+              event: "SUCCESS",
+              status: "ACTIVE",
+            },
+            update: {},
+          });
+        }
+
+        if (
+          headers.data["x-trigger-run-metadata"].failedSubscription &&
+          !run.subscriptions.some((s) => s.event === "FAILURE")
+        ) {
+          await this.#prismaClient.jobRunSubscription.upsert({
+            where: {
+              runId_recipient_event: {
+                runId: run.id,
+                recipient: run.endpoint.id,
+                event: "FAILURE",
+              },
+            },
+            create: {
+              runId: run.id,
+              recipient: run.endpoint.id,
+              recipientMethod: "ENDPOINT",
+              event: "FAILURE",
+              status: "ACTIVE",
+            },
+            update: {},
+          });
+        }
+      }
+
       const rawBody = await response.text();
 
       if (!response.ok) {
@@ -1220,6 +1293,7 @@ async function findRun(prisma: PrismaClientOrTransaction, id: string) {
           status: true,
           noop: true,
           output: true,
+          outputIsUndefined: true,
           parentId: true,
         },
         orderBy: {
@@ -1233,6 +1307,11 @@ async function findRun(prisma: PrismaClientOrTransaction, id: string) {
           organization: true,
         },
       },
+      subscriptions: {
+        where: {
+          recipientMethod: "ENDPOINT",
+        },
+      },
       _count: {
         select: {
           tasks: true,

apps/webapp/app/services/tasks/performTaskOperation.server.ts

@@ -1,20 +1,26 @@
 import {
   FetchOperationSchema,
+  FetchPollOperationSchema,
   FetchRequestInit,
   FetchRetryOptions,
   FetchRetryStrategy,
   RedactString,
   RetryOptions,
+  calculateResetAt,
   calculateRetryAt,
+  eventFilterMatches,
+  responseFilterMatches,
 } from "@trigger.dev/core";
 import { type Task } from "@trigger.dev/database";
 import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
 import { formatUnknownError } from "~/utils/formatErrors.server";
 import { safeJsonFromResponse } from "~/utils/json";
 import { logger } from "../logger.server";
-import { workerQueue } from "../worker.server";
+import { taskOperationWorker, workerQueue } from "../worker.server";
 import { ResumeTaskService } from "./resumeTask.server";
 import { fetch } from "@whatwg-node/fetch";
+import { fromZodError } from "zod-validation-error";
+import { ulid } from "../ulid.server";
 
 type FoundTask = Awaited<ReturnType<typeof findTask>>;
 
@@ -32,16 +38,166 @@ export class PerformTaskOperationService {
       return;
     }
 
+    if (task.status === "CANCELED") {
+      return;
+    }
+
     if (task.status === "COMPLETED" || task.status === "ERRORED") {
       return await this.#resumeRunExecution(task, this.#prismaClient);
     }
 
     if (!task.operation) {
-      return await this.#resumeTask(task, null, 0);
+      return await this.#resumeTask(task, null, null, 200, "fetch", 0);
     }
 
     switch (task.operation) {
-      case "fetch": {
+      case "fetch-poll": {
+        const pollOperation = FetchPollOperationSchema.safeParse(task.params);
+
+        if (!pollOperation.success) {
+          return await this.#resumeTaskWithError(
+            task,
+            fromZodError(pollOperation.error, {
+              prefix: "Invalid fetch poll params",
+            }).message
+          );
+        }
+
+        const { url, requestInit, timeout, interval, responseFilter, requestTimeout } =
+          pollOperation.data;
+
+        // check if we need to fail the task because it's timed out
+        const startedAt = task.startedAt;
+
+        if (!startedAt) {
+          return await this.#resumeTaskWithError(task, {
+            message: "Task has not been started",
+          });
+        }
+
+        if (Date.now() - startedAt.getTime() > timeout * 1000) {
+          return await this.#resumeTaskWithError(task, {
+            message: `Task timed out after ${timeout} seconds`,
+          });
+        }
+
+        const startTimeInMs = performance.now();
+
+        const abortController = new AbortController();
+
+        // calculate the actual timeout. If timeoutInMs is undefined, we use the default of 5s
+        // Also make sure the timeout is at least 1s, but not bigger than 5s
+        const actualTimeoutInMs = Math.min(
+          Math.max(requestTimeout?.durationInMs ?? 5000, 1000),
+          5000
+        );
+
+        const timeoutId = setTimeout(() => {
+          abortController.abort();
+        }, actualTimeoutInMs);
+
+        try {
+          logger.debug("PerformTaskOperationService.call poll request", {
+            task,
+            actualTimeoutInMs,
+            url,
+            responseFilter,
+          });
+
+          const startedAt = new Date();
+
+          const method = requestInit?.method ?? "GET";
+
+          const response = await fetch(url, {
+            method,
+            headers: normalizeHeaders(requestInit?.headers ?? {}),
+            body: requestInit?.body,
+            signal: abortController.signal,
+          });
+
+          clearTimeout(timeoutId);
+
+          const durationInMs = Math.floor(performance.now() - startTimeInMs);
+
+          const headers = Object.fromEntries(response.headers.entries());
+
+          logger.debug("PerformTaskOperationService.call poll response", {
+            url,
+            requestInit,
+            statusCode: response.status,
+            headers: Object.fromEntries(response.headers.entries()),
+            durationInMs,
+          });
+
+          const matchResult = await responseFilterMatches(response, responseFilter);
+
+          await this.#prismaClient.task.create({
+            data: {
+              id: ulid(),
+              idempotencyKey: ulid(),
+              runId: task.runId,
+              parentId: task.id,
+              name: "poll attempt",
+              icon: "activity",
+              status: "COMPLETED",
+              noop: true,
+              style: { style: "minimal", variant: "info" },
+              description: `${method} ${url} ${response.status}`,
+              params: {
+                status: response.status,
+                headers,
+                body: matchResult.body as any,
+              },
+              startedAt,
+              completedAt: new Date(),
+            },
+          });
+
+          if (matchResult.match) {
+            logger.debug("PerformTaskOperationService.call poll response matched", {
+              url,
+              matchResult,
+            });
+
+            return await this.#resumeTask(
+              task,
+              matchResult.body,
+              Object.fromEntries(response.headers.entries()),
+              response.status,
+              "fetch",
+              durationInMs
+            );
+          } else {
+            const retryAt = new Date(Date.now() + interval * 1000);
+
+            return await this.#retryTask(task, retryAt);
+          }
+        } catch (error) {
+          if (error instanceof Error && error.name === "AbortError") {
+            const durationInMs = Math.floor(performance.now() - startTimeInMs);
+
+            logger.debug("PerformTaskOperationService.call poll timed out", {
+              url,
+              durationInMs,
+              error,
+            });
+
+            const retryAt = this.#calculateRetryForTimeout(task, requestTimeout?.retry);
+
+            if (retryAt) {
+              return await this.#retryTask(task, retryAt);
+            }
+
+            return await this.#resumeTaskWithError(task, {
+              message: `Fetch timed out after ${actualTimeoutInMs.toFixed(0)}ms`,
+            });
+          }
+
+          throw error;
+        }
+      }
+      case "fetch":
+      case "fetch-response": {
         const fetchOperation = FetchOperationSchema.safeParse(task.params);
 
         if (!fetchOperation.success) {
@@ -97,7 +253,7 @@ export class PerformTaskOperationService {
           });
 
           if (!response.ok) {
-            const retryAt = this.#calculateRetryForResponse(task, retry, response);
+            const retryAt = this.#calculateRetryForResponse(task, retry, response, jsonBody);
 
             if (retryAt) {
               return await this.#retryTaskWithError(
@@ -117,7 +273,14 @@ export class PerformTaskOperationService {
             }
           }
 
-          return await this.#resumeTask(task, jsonBody, durationInMs);
+          return await this.#resumeTask(
+            task,
+            jsonBody,
+            Object.fromEntries(response.headers.entries()),
+            response.status,
+            task.operation,
+            durationInMs
+          );
         } catch (error) {
           if (error instanceof Error && error.name === "AbortError") {
             const durationInMs = Math.floor(performance.now() - startTimeInMs);
@@ -157,13 +320,14 @@ export class PerformTaskOperationService {
   #calculateRetryForResponse(
     task: NonNullable<FoundTask>,
     retry: FetchRetryOptions | undefined,
-    response: Response
+    response: Response,
+    body: any
   ): Date | undefined {
     if (!retry) {
       return;
     }
 
-    const strategy = this.#getRetryStrategyForStatusCode(response.status, retry);
+    const strategy = this.#getRetryStrategyForResponse(response, body, retry);
 
     if (!strategy) {
       return;
@@ -180,11 +344,10 @@ export class PerformTaskOperationService {
         return calculateRetryAt(strategy, task.attempts.length - 1);
       }
       case "headers": {
-        const remaining = response.headers.get(strategy.remainingHeader);
         const resetAt = response.headers.get(strategy.resetHeader);
 
-        if (typeof remaining === "string" && typeof resetAt === "string" && remaining === "0") {
-          return new Date(Number(resetAt) * 1000 + addJitterInMs());
+        if (typeof resetAt === "string") {
+          return calculateResetAt(resetAt, strategy.resetFormat);
         }
       }
     }
@@ -201,8 +364,9 @@ export class PerformTaskOperationService {
     return calculateRetryAt(retry, task.attempts.length - 1);
   }
 
-  #getRetryStrategyForStatusCode(
-    statusCode: number,
+  #getRetryStrategyForResponse(
+    response: Response,
+    body: any,
     retry: FetchRetryOptions
   ): FetchRetryStrategy | undefined {
     const statusCodes = Object.keys(retry);
@@ -211,7 +375,19 @@ export class PerformTaskOperationService {
       const statusRange = statusCodes[i];
       const strategy = retry[statusRange];
 
-      if (isStatusCodeInRange(statusCode, statusRange)) {
+      if (isStatusCodeInRange(response.status, statusRange)) {
+        if (strategy.bodyFilter) {
+          if (!body) {
+            continue;
+          }
+
+          if (eventFilterMatches(body, strategy.bodyFilter)) {
+            return strategy;
+          } else {
+            continue;
+          }
+        }
+
         return strategy;
       }
     }
@@ -248,16 +424,26 @@ export class PerformTaskOperationService {
         },
       });
 
-      await workerQueue.enqueue(
+      await taskOperationWorker.enqueue(
         "performTaskOperation",
         {
           id: task.id,
         },
-        { tx, runAt: retryAt }
+        { tx, runAt: retryAt, jobKey: `operation:${task.id}` }
       );
     });
   }
 
+  async #retryTask(task: Task, retryAt: Date) {
+    await taskOperationWorker.enqueue(
+      "performTaskOperation",
+      {
+        id: task.id,
+      },
+      { runAt: retryAt, jobKey: `operation:${task.id}` }
+    );
+  }
+
   async #resumeTaskWithError(task: NonNullable<FoundTask>, output: any) {
     await $transaction(this.#prismaClient, async (tx) => {
       await tx.task.update({
@@ -284,7 +470,14 @@ export class PerformTaskOperationService {
     });
   }
 
-  async #resumeTask(task: NonNullable<FoundTask>, output: any, durationInMs: number) {
+  async #resumeTask(
+    task: NonNullable<FoundTask>,
+    output: any,
+    context: any,
+    status: number,
+    operation: "fetch" | "fetch-response",
+    durationInMs: number
+  ) {
     await $transaction(this.#prismaClient, async (tx) => {
       await tx.taskAttempt.updateMany({
         where: {
@@ -296,12 +489,22 @@ export class PerformTaskOperationService {
         },
       });
 
+      const taskOutput =
+        operation === "fetch"
+          ? output
+          : {
+              data: output,
+              headers: context,
+              status,
+            };
+
       await tx.task.update({
         where: { id: task.id },
         data: {
           status: "COMPLETED",
           completedAt: new Date(),
-          output: output ? output : undefined,
+          output: taskOutput,
+          context: context ? context : undefined,
           run: {
             update: {
               executionDuration: {

apps/webapp/app/services/tasks/runTask.server.ts

@@ -0,0 +1,220 @@
+import { RunTaskBodyOutput, ServerTask } from "@trigger.dev/core";
+import { TaskStatus } from "@trigger.dev/database";
+import { $transaction, PrismaClient, prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { taskWithAttemptsToServerTask } from "~/models/task.server";
+import { generateSecret } from "~/services/sources/utils.server";
+import { ulid } from "~/services/ulid.server";
+import { taskOperationWorker, workerQueue } from "~/services/worker.server";
+
+export class RunTaskService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    runId: string,
+    idempotencyKey: string,
+    taskBody: RunTaskBodyOutput
+  ): Promise<ServerTask | undefined> {
+    const delayUntilInFuture = taskBody.delayUntil
+      ? taskBody.delayUntil.getTime() > Date.now()
+      : false;
+    const callbackEnabled = taskBody.callback?.enabled ?? false;
+
+    // First
+    const existingTask = await this.#handleExistingTask(
+      runId,
+      idempotencyKey,
+      taskBody,
+      delayUntilInFuture,
+      callbackEnabled
+    );
+
+    if (existingTask) {
+      return taskWithAttemptsToServerTask(existingTask);
+    }
+
+    const run = await this.#prismaClient.jobRun.findUnique({
+      where: {
+        id: runId,
+      },
+      select: {
+        status: true,
+        forceYieldImmediately: true,
+      },
+    });
+
+    if (!run) throw new Error("Run not found");
+
+    const runConnection = taskBody.connectionKey
+      ? await this.#prismaClient.runConnection.findUnique({
+          where: {
+            runId_key: {
+              runId,
+              key: taskBody.connectionKey,
+            },
+          },
+          select: {
+            id: true,
+          },
+        })
+      : undefined;
+
+    const results = await $transaction(this.#prismaClient, async (tx) => {
+      // If task.delayUntil is set and is in the future, we'll set the task's status to "WAITING", else set it to RUNNING
+      let status: TaskStatus;
+
+      if (run.status === "CANCELED") {
+        status = "CANCELED";
+      } else {
+        status =
+          delayUntilInFuture || callbackEnabled || taskBody.trigger
+            ? "WAITING"
+            : taskBody.noop
+            ? "COMPLETED"
+            : "RUNNING";
+      }
+
+      const taskId = ulid();
+      const callbackUrl = callbackEnabled
+        ? `${env.APP_ORIGIN}/api/v1/tasks/${taskId}/callback/${generateSecret(12)}`
+        : undefined;
+
+      const task = await tx.task.create({
+        data: {
+          id: taskId,
+          idempotencyKey,
+          displayKey: taskBody.displayKey,
+          runConnectionId: runConnection ? runConnection.id : undefined,
+          icon: taskBody.icon,
+          runId,
+          parentId: taskBody.parentId,
+          name: taskBody.name ?? "Task",
+          description: taskBody.description,
+          status,
+          startedAt: new Date(),
+          completedAt: status === "COMPLETED" || status === "CANCELED" ? new Date() : undefined,
+          noop: taskBody.noop,
+          delayUntil: taskBody.delayUntil,
+          params: taskBody.params ?? undefined,
+          properties: this.#filterProperties(taskBody.properties) ?? undefined,
+          redact: taskBody.redact ?? undefined,
+          operation: taskBody.operation,
+          callbackUrl,
+          style: taskBody.style ?? { style: "normal" },
+          childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
+        },
+      });
+
+      const taskAttempt = await tx.taskAttempt.create({
+        data: {
+          number: 1,
+          taskId: task.id,
+          status: "PENDING",
+        },
+      });
+
+      if (task.status === "RUNNING" && typeof taskBody.operation === "string") {
+        // We need to schedule the operation
+        await taskOperationWorker.enqueue(
+          "performTaskOperation",
+          {
+            id: task.id,
+          },
+          { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
+        );
+      } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
+        if (taskBody.callback.timeoutInSeconds > 0) {
+          // We need to schedule the callback timeout
+          await workerQueue.enqueue(
+            "processCallbackTimeout",
+            {
+              id: task.id,
+            },
+            {
+              tx,
+              runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
+              jobKey: `process-callback:${task.id}`,
+            }
+          );
+        }
+      }
+
+      return { task, taskAttempt };
+    });
+
+    if (!results) {
+      return;
+    }
+
+    const { task, taskAttempt } = results;
+
+    return task
+      ? taskWithAttemptsToServerTask({ ...task, attempts: [taskAttempt], run })
+      : undefined;
+  }
+
+  async #handleExistingTask(
+    runId: string,
+    idempotencyKey: string,
+    taskBody: RunTaskBodyOutput,
+    delayUntilInFuture: boolean,
+    callbackEnabled: boolean
+  ) {
+    const existingTask = await this.#prismaClient.task.findUnique({
+      where: {
+        runId_idempotencyKey: {
+          runId,
+          idempotencyKey,
+        },
+      },
+      include: {
+        attempts: true,
+        run: true,
+      },
+    });
+
+    if (existingTask) {
+      if (existingTask.status === "CANCELED") {
+        const existingTaskStatus =
+          delayUntilInFuture || callbackEnabled || taskBody.trigger
+            ? "WAITING"
+            : taskBody.noop
+            ? "COMPLETED"
+            : "RUNNING";
+
+        const resumedExistingTask = await this.#prismaClient.task.update({
+          where: {
+            id: existingTask.id,
+          },
+          data: {
+            status: existingTaskStatus,
+            startedAt: new Date(),
+            completedAt: existingTaskStatus === "COMPLETED" ? new Date() : undefined,
+          },
+          include: {
+            run: true,
+            attempts: true,
+          },
+        });
+
+        return resumedExistingTask;
+      }
+
+      return existingTask;
+    }
+  }
+
+  #filterProperties(properties: RunTaskBodyOutput["properties"]): RunTaskBodyOutput["properties"] {
+    if (!properties) return;
+
+    return properties.filter((property) => {
+      if (!property) return false;
+
+      return typeof property.label === "string" && typeof property.text === "string";
+    });
+  }
+}

apps/webapp/app/services/worker.server.ts

@@ -24,6 +24,8 @@ import { ProbeEndpointService } from "./endpoints/probeEndpoint.server";
 import { DeliverRunSubscriptionService } from "./runs/deliverRunSubscription.server";
 import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.server";
 import { ResumeTaskService } from "./tasks/resumeTask.server";
+import { ExpireDispatcherService } from "./dispatchers/expireDispatcher.server";
+import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralEventDispatcher.server";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -40,9 +42,6 @@ const workerCatalog = {
   processCallbackTimeout: z.object({
     id: z.string(),
   }),
-  performTaskOperation: z.object({
-    id: z.string(),
-  }),
   deliverHttpSourceRequest: z.object({ id: z.string() }),
   refreshOAuthToken: z.object({
     organizationId: z.string(),
@@ -93,6 +92,9 @@ const workerCatalog = {
   resumeTask: z.object({
     id: z.string(),
   }),
+  expireDispatcher: z.object({
+    id: z.string(),
+  }),
 };
 
 const executionWorkerCatalog = {
@@ -108,12 +110,24 @@ const executionWorkerCatalog = {
   }),
 };
 
+const taskOperationWorkerCatalog = {
+  performTaskOperation: z.object({
+    id: z.string(),
+  }),
+  invokeEphemeralDispatcher: z.object({
+    id: z.string(),
+    eventRecordId: z.string(),
+  }),
+};
+
 let workerQueue: ZodWorker<typeof workerCatalog>;
 let executionWorker: ZodWorker<typeof executionWorkerCatalog>;
+let taskOperationWorker: ZodWorker<typeof taskOperationWorkerCatalog>;
 
 declare global {
   var __worker__: ZodWorker<typeof workerCatalog>;
   var __executionWorker__: ZodWorker<typeof executionWorkerCatalog>;
+  var __taskOperationWorker__: ZodWorker<typeof taskOperationWorkerCatalog>;
 }
 
 // this is needed because in development we don't want to restart
@@ -123,6 +137,7 @@ declare global {
 if (env.NODE_ENV === "production") {
   workerQueue = getWorkerQueue();
   executionWorker = getExecutionWorkerQueue();
+  taskOperationWorker = getTaskOperationWorkerQueue();
 } else {
   if (!global.__worker__) {
     global.__worker__ = getWorkerQueue();
@@ -134,6 +149,12 @@ if (env.NODE_ENV === "production") {
   }
 
   executionWorker = global.__executionWorker__;
+
+  if (!global.__taskOperationWorker__) {
+    global.__taskOperationWorker__ = getTaskOperationWorkerQueue();
+  }
+
+  taskOperationWorker = global.__taskOperationWorker__;
 }
 
 export async function init() {
@@ -148,6 +169,10 @@ export async function init() {
   if (env.EXECUTION_WORKER_ENABLED === "true") {
     await executionWorker.initialize();
   }
+
+  if (env.TASK_OPERATION_WORKER_ENABLED === "true") {
+    await taskOperationWorker.initialize();
+  }
 }
 
 function getWorkerQueue() {
@@ -286,15 +311,6 @@ function getWorkerQueue() {
           await service.call(payload.id);
         },
       },
-      performTaskOperation: {
-        priority: 0, // smaller number = higher priority
-        maxAttempts: 3,
-        handler: async (payload, job) => {
-          const service = new PerformTaskOperationService();
-
-          await service.call(payload.id);
-        },
-      },
       scheduleEmail: {
         priority: 100,
         maxAttempts: 3,
@@ -375,6 +391,15 @@ function getWorkerQueue() {
         handler: async (payload, job) => {
           const service = new ResumeTaskService();
 
+          return await service.call(payload.id);
+        },
+      },
+      expireDispatcher: {
+        priority: 10,
+        maxAttempts: 3,
+        handler: async (payload) => {
+          const service = new ExpireDispatcherService();
+
           return await service.call(payload.id);
         },
       },
@@ -417,15 +442,57 @@ function getExecutionWorkerQueue() {
         handler: async (payload, job) => {
           const service = new PerformRunExecutionV3Service();
 
-          await service.call({
-            id: payload.id,
-            reason: payload.reason,
-            isRetry: false,
-          });
+          const driftInMs = Date.now() - job.run_at.getTime();
+
+          await service.call(
+            {
+              id: payload.id,
+              reason: payload.reason,
+              isRetry: false,
+            },
+            driftInMs
+          );
         },
       },
     },
   });
 }
 
-export { executionWorker, workerQueue };
+function getTaskOperationWorkerQueue() {
+  return new ZodWorker({
+    name: "taskOperationWorker",
+    prisma,
+    runnerOptions: {
+      connectionString: env.DATABASE_URL,
+      concurrency: env.TASK_OPERATION_WORKER_CONCURRENCY,
+      pollInterval: env.TASK_OPERATION_WORKER_POLL_INTERVAL,
+      noPreparedStatements: env.DATABASE_URL !== env.DIRECT_URL,
+      schema: env.WORKER_SCHEMA,
+      maxPoolSize: env.TASK_OPERATION_WORKER_CONCURRENCY,
+    },
+    shutdownTimeoutInMs: env.GRACEFUL_SHUTDOWN_TIMEOUT,
+    schema: taskOperationWorkerCatalog,
+    tasks: {
+      performTaskOperation: {
+        priority: 0, // smaller number = higher priority
+        maxAttempts: 3,
+        handler: async (payload, job) => {
+          const service = new PerformTaskOperationService();
+
+          await service.call(payload.id);
+        },
+      },
+      invokeEphemeralDispatcher: {
+        priority: 0, // smaller number = higher priority
+        maxAttempts: 10,
+        handler: async (payload, job) => {
+          const service = new InvokeEphemeralDispatcherService();
+
+          await service.call(payload.id, payload.eventRecordId);
+        },
+      },
+    },
+  });
+}
+
+export { executionWorker, workerQueue, taskOperationWorker };

apps/webapp/app/utils/pathBuilder.ts

@@ -369,3 +369,8 @@ export function docsIntegrationPath(api: string) {
 export function docsCreateIntegration() {
   return `${docsRoot()}/integrations/create`;
 }
+
+//api
+export function apiReferencePath(apiSlug: string) {
+  return `https://trigger.dev/apis/${apiSlug}`;
+}

apps/webapp/app/utils/sse.server.ts

@@ -1,4 +1,5 @@
 import { eventStream } from "remix-utils/sse/server";
+import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
 
 type SseProps = {
@@ -17,20 +18,21 @@ type Event = {
 };
 
 export function sse({ request, pingInterval = 1000, updateInterval = 348, run }: SseProps) {
+  if (env.DISABLE_SSE === "1" || env.DISABLE_SSE === "true") {
+    return new Response("SSE disabled", { status: 200 });
+  }
+
   let pinger: NodeJS.Timer | undefined = undefined;
   let updater: NodeJS.Timer | undefined = undefined;
+  let timeout: NodeJS.Timeout | undefined = undefined;
 
   const abort = () => {
-    if (pinger) {
-      clearInterval(pinger);
-    }
-
-    if (updater) {
-      clearInterval(updater);
-    }
+    clearInterval(pinger);
+    clearInterval(updater);
+    clearTimeout(timeout);
   };
 
-  return eventStream(request.signal, (send) => {
+  return eventStream(request.signal, (send, close) => {
     const safeSend = (args: { event?: string; data: string }) => {
       try {
         send(args);
@@ -53,18 +55,30 @@ export function sse({ request, pingInterval = 1000, updateInterval = 348, run }:
           });
         }
 
-        abort();
+        close();
       }
     };
 
     pinger = setInterval(() => {
+      if (request.signal.aborted) {
+        return abort();
+      }
+
       safeSend({ event: "ping", data: new Date().toISOString() });
     }, pingInterval);
 
-    updater = setInterval(async () => {
+    updater = setInterval(() => {
+      if (request.signal.aborted) {
+        return abort();
+      }
+
       run(safeSend, abort);
     }, updateInterval);
 
+    timeout = setTimeout(() => {
+      close(); // close the connection after 1 minute of inactivity, which will refresh the connection (that's why we aren't using abort)
+    }, 60 * 1000); // 1 minute
+
     return abort;
   });
 }

apps/webapp/package.json

@@ -29,6 +29,7 @@
     "/public/build"
   ],
   "dependencies": {
+    "@aws-sdk/client-sqs": "^3.445.0",
     "@codemirror/autocomplete": "^6.3.1",
     "@codemirror/commands": "^6.1.2",
     "@codemirror/lang-javascript": "^6.1.1",
@@ -62,7 +63,7 @@
     "@remix-run/v1-meta": "^0.1.3",
     "@tabler/icons-react": "^2.39.0",
     "@team-plain/typescript-sdk": "^2.2.0",
-    "@trigger.dev/companyicons": "^1.5.14",
+    "@trigger.dev/companyicons": "^1.5.32",
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/core-backend": "workspace:*",
     "@trigger.dev/database": "workspace:*",
@@ -102,7 +103,7 @@
     "react-use": "^17.4.0",
     "recharts": "^2.8.0",
     "remix-auth": "^3.6.0",
-    "remix-auth-email-link": "^2.1.0",
+    "remix-auth-email-link": "2.0.2",
     "remix-auth-github": "^1.6.0",
     "remix-typedjson": "0.3.1",
     "remix-utils": "^7.1.0",
@@ -111,6 +112,7 @@
     "simplur": "^3.0.1",
     "slug": "^6.0.0",
     "sonner": "^1.0.3",
+    "sqs-consumer": "^7.4.0",
     "tailwind-merge": "^1.12.0",
     "tailwind-scrollbar-hide": "^1.1.7",
     "tailwindcss-animate": "^1.0.5",

apps/webapp/server.ts

@@ -20,7 +20,9 @@ app.use((req, res, next) => {
   next();
 });
 
-app.use(compression());
+if (process.env.DISABLE_COMPRESSION !== "1") {
+  app.use(compression());
+}
 
 // http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header
 app.disable("x-powered-by");

docs/_snippets/beta-feature-warning.mdx

@@ -0,0 +1 @@
+<Warning>This feature is in beta and has not yet been deployed to the Trigger.dev cloud</Warning>

docs/_snippets/jobs/options.mdx

@@ -36,6 +36,12 @@
     <ParamField body="enabled" type="boolean">
       The `enabled` property is an optional property that specifies whether the Job is enabled or not. The Job will be enabled by default if you omit this property. When a job is disabled, no new runs will be triggered or resumed. In progress runs will continue to run until they are finished or delayed by using `io.wait`.
     </ParamField>
+    <ParamField body="onSuccess" type="function">
+      The `onSuccess` property is an optional property that specifies a callback function to run when the Job finishes successfully. The callback function receives a [Run Notification](/sdk/run-notification) object as it's only parameter.
+    </ParamField>
+    <ParamField body="onFailure" type="function">
+      The `onFailure` property is an optional property that specifies a callback function to run when the Job fails to complete successfully. The callback function receives a [Run Notification](/sdk/run-notification) object as it's only parameter.
+    </ParamField>
     <ParamField body="logLevel" type="log | error | warn | info | debug">
       The `logLevel` property is an optional property that specifies the level of
       logging for the Job. The level is inherited from the client if you omit this property.

docs/_snippets/send-events-params.mdx

@@ -0,0 +1,52 @@
+<ParamField body="events" type="array" required>
+  <Expandable title="event properties" defaultOpen>
+    <ParamField body="name" type="string" required>
+      The `name` property must exactly match any subscriptions you want to
+      trigger.
+    </ParamField>
+    <ParamField body="payload" type="any">
+      The `payload` property will be sent to any matching Jobs and will appear
+      as the `payload` param of the `run()` function. You can leave this
+      parameter out if you just want to trigger a Job without any input data.
+    </ParamField>
+    <ParamField body="context" type="any">
+      The optional `context` property will be sent to any matching Jobs and will
+      be passed through as the `context.event.context` param of the `run()`
+      function. This is optional but can be useful if you want to pass through
+      some additional context to the Job.
+    </ParamField>
+    <ParamField body="id" type="string">
+      The `id` property uniquely identify this particular event. If unset it
+      will be set automatically using `ulid`.
+    </ParamField>
+    <ParamField body="timestamp" type="Date">
+      This is optional, it defaults to the current timestamp. Usually you would
+      only set this if you have a timestamp that you wish to pass through, e.g.
+      you receive a timestamp from a service and you want the same timestamp to
+      be used in your Job.
+    </ParamField>
+    <ParamField body="source" type="string">
+      This is optional, it defaults to "trigger.dev". It can be useful to set
+      this as you can filter events using this in the `eventTrigger()`.
+    </ParamField>
+  </Expandable>
+</ParamField>
+
+<ParamField body="options" type="object">
+  <Expandable title="properties" defaultOpen>
+    <ParamField body="deliverAt" type="Date">
+      An optional Date when you want the event to Trigger Jobs. The event will
+      be sent to the platform immediately but won't be acted upon until the
+      specified time.
+    </ParamField>
+    <ParamField body="deliverAfter" type="number">
+      An optional number of seconds you want to wait for the event to Trigger
+      any relevant Jobs. The event will be sent to the platform immediately but
+      won't be acted upon until the specified time.
+    </ParamField>
+    <ParamField body="accountId" type="string">
+      This optional param will be used by the Trigger.dev Connect feature, which
+      is coming soon.
+    </ParamField>
+  </Expandable>
+</ParamField>

docs/_snippets/send-events-return.mdx

@@ -0,0 +1,29 @@
+<ResponseField name="events" type="array">
+  <Expandable title="properties" defaultOpen>
+    <ResponseField name="id" type="string" required>
+      The `id` of the event that was sent.
+    </ResponseField>
+    <ResponseField name="name" type="string" required>
+      The `name` of the event that was sent.
+    </ResponseField>
+    <ResponseField name="payload" type="any" required>
+      The `payload` of the event that was sent
+    </ResponseField>
+    <ResponseField name="timestamp" type="Date" required>
+      The `timestamp` of the event that was sent
+    </ResponseField>
+    <ResponseField name="context" type="any">
+      The `context` of the event that was sent. Is `undefined` if no context was
+      set when sending the event.
+    </ResponseField>
+    <ResponseField name="deliverAt" type="Date">
+      The timestamp when the event will be delivered to any matching Jobs. Is
+      `undefined` if `deliverAt` or `deliverAfter` wasn't set when sending the
+      event.
+    </ResponseField>
+    <ResponseField name="deliveredAt" type="Date">
+      The timestamp when the event was delivered. Is `undefined` if `deliverAt`
+      or `deliverAfter` were set when sending the event.
+    </ResponseField>
+  </Expandable>
+</ResponseField>

docs/documentation/concepts/http-endpoints.mdx

@@ -3,13 +3,11 @@ title: HTTP endpoints
 description: HTTP endpoints allow you to trigger your Jobs from any webhooks.
 ---
 
-<Warning>This feature is in beta and not yet available to use on the Trigger.dev Cloud.</Warning>
-
 Sometimes you want to subscribe to changes from an API, and we don't have [an Integration](/integrations/introduction) for it yet. That's when you can use `defineHttpEndpoint` to receive webhooks, verify them, and create an [HTTP Trigger](/documentation/concepts/triggers/http).
 
 ## Defining an HTTP endpoint
 
-Defining an HTTP endpoint creates a URL and secret which you'll enter into Cal.com's website. It also attaches a `verify` function that is called when a webhook is received. It's compulsory to return a result from this function – 90% of the time you can use our `verifyRequestSignature` helper function.
+We'll use Cal.com as an example:
 
 ```ts
 const caldotcom = client.defineHttpEndpoint({
@@ -35,6 +33,10 @@ const caldotcom = client.defineHttpEndpoint({
 });
 ```
 
+When this code runs (and you're running the CLI dev command) the HTTP endpoint will be created and be visible in the Trigger.dev dashboard.
+
+The `verify` function is compulsory and is automatically called when a webhook is received. It's required to return a result from this function – 90% of the time you can use our `verifyRequestSignature` helper function.
+
 ## Getting the URL and secret
 
 In our dashboard, you can navigate to the HTTP endpoints page. From there you can select your endpoint and copy the URL (1) and secret (2) for the appropriate Environment.

docs/documentation/concepts/triggers/http.mdx

@@ -4,8 +4,6 @@ sidebarTitle: "HTTP"
 description: "HTTP Triggers allow you to trigger your Jobs from any webhooks."
 ---
 
-<Warning>This feature is in beta and not yet available to use on the Trigger.dev Cloud.</Warning>
-
 Sometimes you want to subscribe to changes from an API, and we don't have [an Integration](/integrations/introduction) for it yet. That's when you can use `defineHttpEndpoint` to receive webhooks, verify them, and create an HTTP Trigger.
 
 You should read the [HTTP endpoint](/documentation/concepts/http-endpoints) documentation to understand how to create an HTTP endpoint.

docs/documentation/concepts/triggers/invoke.mdx

@@ -4,8 +4,6 @@ sidebarTitle: "Manual Invoke"
 description: "Invoke Jobs manually using the invoke Trigger"
 ---
 
-<Warning>This feature is in beta and not yet available to use on the Trigger.dev Cloud.</Warning>
-
 Sometimes it makes sense to be able to invoke a Job manually, without having to specify an event, especially for cases where you want to get notified when the invoked Job Run is complete.
 
 To specify that a job is manually invokable, you can use the `invokeTrigger()` function when defining a job:

docs/documentation/guides/task-library.mdx

@@ -0,0 +1,255 @@
+---
+title: "Task Library"
+description: "These are the built-in tasks that are available to use in your Jobs."
+---
+
+Welcome to the Trigger.dev Task Library 📚. You may be wondering, what is a Task and why are there a library of them? Well you see, Trigger.dev works by divvying up a long-running job execution into a bunch of little tasks, each one taking less time then a single serverless function execution. 💫
+
+You can define and run your own tasks easily using [io.runTask()](/sdk/io/runtask), or you can use one of our [Integrations](/integrations/introduction) which are tasks for specific APIs, like OpenAI or Stripe.
+
+<Note>Read more about how Tasks work [here](/documentation/concepts/tasks).</Note>
+
+We also have a growing library of built-in tasks that you can use in your Jobs through the `io` object. These tasks are designed to be generic and reusable, and are a great way to get started with Trigger.dev.
+
+<Info>
+  You may notice that I'm using emojis for all the cache keys below, which is totally 💯% fine as
+  long as they are unique inside a run. Read more about how cache keys work
+  [here](/documentation/concepts/tasks#task-cache-keys)
+</Info>
+
+## `wait`
+
+This task allows you to resume executing your job after a certain amount of time has passed:
+
+```ts
+await io.wait("⏰", 60); // wait 60 seconds
+```
+
+Internally this task is considered a "noop", and noop tasks have no output.
+
+[reference docs](/sdk/io/wait)
+
+## `waitForRequest`
+
+You supply this task with a callback to receive a URL. When a POST request is made to that URL, the JSON body of the request becomes the task output.
+
+The example below uses `waitForRequest` to capture a Screenshot of a website using [ScreenshotOne.com](https://screenshotone.com/) and passes the callback URL to the webhook URL to get notified when the screenshot is finished:
+
+```ts
+const result = await io.waitForRequest<ScreenshotResponse>(
+  "📸",
+  async (url) => {
+    await fetch(`https://api.screenshotone.com/take`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        webhook_url: url, // this is the URL that will be called when the screenshot is ready
+        access_key: "my-access-key",
+        url: "https://trigger.dev",
+        store: "true",
+        storage_path: "my-screeshots",
+        response_type: "json",
+        async: "true",
+        storage_return_location: "true",
+      }),
+    });
+  },
+  {
+    timeoutInSeconds: 300, // wait up to 5 minutes for the screenshot to be ready
+  }
+);
+```
+
+We actually originally built this task for our [Replicate integration](/integrations/apis/replicate), which accepts a callback URL to notify you when a prediction is ready. So this allows you to write very succinct code to create a prediction and wait for it's results:
+
+```ts
+const sdPrediction = await io.replicate.predictions.createAndAwait("🧑‍🎨", {
+  version: "ac732df83cea7fff18b8472768c88ad041fa750ff7682a21affe81863cbe77e4",
+  input: {
+    prompt: "What is the meaning of life?",
+  },
+});
+```
+
+[reference docs](/sdk/io/wait-for-request)
+
+## `waitForEvent`
+
+This task allows you to wait for an event to be sent. To read about how events work, check out the [Events](/documentation/concepts/triggers/events) documentation.
+
+```ts
+const event = await io.waitForEvent(
+  "🥂",
+  {
+    name: "user.created",
+    schema: z.object({
+      id: z.string(),
+      createdAt: z.coerce.date(),
+      isAdmin: z.boolean(),
+    }),
+    filter: {
+      isAdmin: [true], // Only wait for events where isAdmin is true
+    },
+  },
+  {
+    timeoutInSeconds: 60 * 60, // Wait for up to an hour
+  }
+);
+```
+
+The event object returned from this task is the full event object that was sent, including `id`, `name`, `payload`, `context`, and more.
+
+[reference docs](/sdk/io/wait-for-event)
+
+## `backgroundFetch`
+
+This task allows you to perform a `fetch` request in the background, and then resume the execution of your job after the request has completed.
+
+```ts
+const body = io.backgroundFetch<MyResponseData>("🕸️", "https://example.com/api/endpoint", {
+  method: "POST",
+  headers: {
+    "Content-Type": "application/json",
+    Authorization: redactString`Bearer ${auth.apiKey}`,
+  },
+  body: JSON.stringify({ foo: "bar" }),
+});
+```
+
+This is useful for when an API is slow to respond and might not finish before your serverless function times out. We created this task to power our [OpenAI integration](/integrations/apis/openai), which can sometimes take more than a minute to respond:
+
+```ts
+// This uses backgroundFetch under the hood
+await io.openai.chat.completions.backgroundCreate("💬", {
+  model: "gpt-3.5-turbo",
+  messages: [
+    {
+      role: "user",
+      content: "Create a good programming joke about background jobs",
+    },
+  ],
+});
+```
+
+[reference docs](/sdk/io/backgroundfetch)
+
+## `backgroundPoll`
+
+This task is similar to `backgroundFetch`, but instead of waiting for a single request to complete, it will poll a URL until it returns a certain value.
+
+```ts
+const result = await io.backgroundPoll<{ foo: string }>("🔃", {
+  url: "https://example.com/api/endpoint",
+  interval: 10, // every 10 seconds
+  timeout: 300, // stop polling after 5 minutes
+  responseFilter: {
+    // stop polling once this filter matches
+    status: [200],
+    body: {
+      status: ["SUCCESS"],
+    },
+  },
+});
+```
+
+## `logger`
+
+The logger object allows you to log messages to the Trigger.dev console. This is useful for debugging your jobs, or just to see what's going on inside your job.
+
+```ts
+await io.logger.info("This is an info message");
+```
+
+You can optionally pass a `context` object to the logger, which will be displayed in the console:
+
+```ts
+await io.logger.info("This is an info message", {
+  foo: "bar",
+});
+```
+
+We support the following log levels:
+
+- `io.logger.debug()`
+- `io.logger.info()`
+- `io.logger.warn()`
+- `io.logger.error()`
+
+<Note>
+  You may notice these tasks don't include cache keys. We automatically create a cache key for you
+  based on the message and the log-level
+</Note>
+
+[reference docs](/sdk/io/logger)
+
+## `random`
+
+Use this task to generate a random number that stays stable during run retries/resumes:
+
+```ts
+const randomNumber = await io.random("🎲", {
+  min: 1,
+  max: 100,
+});
+```
+
+[reference docs](/sdk/io/random)
+
+## `sendEvent`
+
+This task allows you to send an event from inside your job run.
+
+If you want to send an event from outside a run (e.g. just from your backend) you should use [client.sendEvent()](/sdk/triggerclient/instancemethods/sendevent) instead.
+
+```ts
+await io.sendEvent("🚚", {
+  id: "e_1234567890",
+  name: "new.user",
+  payload: {
+    userId: "u_1234567890",
+  },
+});
+```
+
+[reference docs](/sdk/io/sendevent)
+
+## `getEvent`
+
+This task allows you to get an event by ID from inside your job run.
+
+If you want to get an event from outside a run (e.g. just from your backend) you should use [client.getEvent()](/sdk/triggerclient/instancemethods/getevent) instead.
+
+```ts
+const event = await io.getEvent("📥", "e_1234567890");
+```
+
+[reference docs](/sdk/io/getevent)
+
+## `cancelEvent`
+
+If you send an event that has a delivery date in the future, you can use this task to cancel it.
+
+```ts
+await io.sendEvent(
+  "🚚",
+  {
+    id: "e_1234567890",
+    name: "new.user",
+    payload: {
+      userId: "u_1234567890",
+    },
+  },
+  {
+    deliverAt: new Date(Date.now() + 1000 * 60 * 60 * 24), // deliver in 24 hours
+  }
+);
+
+// Later on, if you want to cancel the event:
+await io.cancelEvent("🚫", "e_1234567890");
+```
+
+## `createStatus`
+
+Coming soon

docs/documentation/guides/writing-jobs-step-by-step.mdx

@@ -68,7 +68,7 @@ client.defineJob({
 });
 ```
 
-## 5. Add integrations
+## 5. Add integrations (optional)
 
 Integrations provide a convienent way to create and run tasks against authenticated APIs inside your Job's run function. You'll need to pass them in the `integrations` option when defining your Job.
 
@@ -165,6 +165,7 @@ client.defineJob({
   trigger: eventTrigger({
     name: "my.event",
   }),
+  // Optional integration
   integrations: { slack },
   run: async (payload, io, context) => {
     await io.logger.info("Received the my.event event", { payload });
@@ -198,6 +199,7 @@ client.defineJob({
   trigger: eventTrigger({
     name: "my.event",
   }),
+  // Optional integration
   integrations: { slack },
   run: async (payload, io, context) => {
     // Use runTask with the "get-user" cacheKey, and return the user
@@ -366,17 +368,15 @@ We recommend exploring all of the below sections to fully understand how to crea
 
     </Card>
 
- <Card title="Example Jobs" icon="slot-machine" href="/examples">
-    View example Jobs / the example jobs repo. These are a great starting
-    point for creating your own Jobs.
-  </Card>
-  <Card title="SDK reference" icon="code" href="/sdk">
-    How to use the SDK. This includes all the available Tasks, triggers and
-    actions you can use.
-  </Card>
- 
-  <Card title="Integrations" icon="grid-2" href="/integrations">
-    Integrations make it easy to authenticate and use APIs.
-Learn how to use and create integrations.
-  </Card>
+<Card title="Integrations" icon="grid-2" href="/integrations">
+  Integrations make it easy to authenticate and use APIs. Learn how to use and create integrations.
+</Card>
+<Card title="API catalog with code samples" icon="code" href="https://trigger.dev/apis">
+  Find code examples for the API you need that you can copy and paste into your
+  projects.
+</Card>
+<Card title="SDK reference" icon="book-open" href="/sdk">
+  How to use the SDK. This includes all the available Tasks, triggers and actions you can use.
+</Card>
+
 </CardGroup>

docs/examples/introduction.mdx

@@ -1,17 +1,23 @@
 ---
 title: Example Jobs
-description: "Jobs and projects you can use to get started."
+description: "Jobs and code examples you can use to get started."
 ---
 
-<Card
-  title="Browse our Jobs Showcase"
-  icon="rocket-launch"
-  href="https://trigger.dev/showcase"
-  color="#EC4899"
->
-  The showcase is our ever-growing library of Jobs. Use them as they are, right out of the box, or
-  customize them to suit your needs.
-</Card>
+<CardGroup>
+  <Card
+    title="Browse our Jobs Showcase"
+    icon="rocket-launch"
+    href="https://trigger.dev/showcase"
+    color="#EC4899"
+  >
+    The showcase is our library of Jobs. Use them as they are, right out of the box, or customize
+    them to suit your needs.
+  </Card>
+  <Card title="API catalog with code samples" icon="code" href="https://trigger.dev/apis">
+    Find code examples for the API you need that you can copy and paste into your
+  projects.
+  </Card>
+</CardGroup>
 
 ### Full projects
 
@@ -24,6 +30,6 @@ To run them, simply follow the instructions in the README files linked below.
 | [OpenAI text summarizer](https://github.com/triggerdotdev/examples/tree/main/openai-text-summarizer)             | An app which uses OpenAI to summarize an article and then post the result to Slack.                                                         | [OpenAI](https://trigger.dev/docs/integrations/apis/openai) [Slack](https://trigger.dev/docs/integrations/apis/slack)       | Trigger.dev                             | ✅     |
 | [Supabase onboarding emails](https://github.com/triggerdotdev/examples/tree/main/supabase-onboarding-emails)     | When a user signs up and confirms their email address, they will receive 3 "onboarding" emails over 2 days using Resend.com and Trigger.dev | [Supabase](https://trigger.dev/docs/integrations/apis/supabase) [Resend](https://trigger.dev/docs/integrations/apis/resend) | Trigger.dev                             | ✅     |
 | [Generate presentation titles using OpenAI](https://github.com/triggerdotdev/examples/tree/main/express-vanilla) | Generate presentation titles using OpenAI background jobs with Node.js, Express and Trigger.dev                                             | [OpenAI](https://trigger.dev/docs/integrations/apis/openai)                                                                 | [lirantal](https://github.com/lirantal) | ✅     |
-| [Send a basic email with Resend](https://github.com/triggerdotdev/examples/tree/main/resend-email-form)                     | Send a basic email from a form with Resend                                                                                                  | [Resend](https://trigger.dev/docs/integrations/apis/resend)                                                                 | Trigger.dev                             | ✅     |
+| [Send a basic email with Resend](https://github.com/triggerdotdev/examples/tree/main/resend-email-form)          | Send a basic email from a form with Resend                                                                                                  | [Resend](https://trigger.dev/docs/integrations/apis/resend)                                                                 | Trigger.dev                             | ✅     |
 | AI landing page copy generator                                                                                   | Copies your site and generates new copy using OpenAI                                                                                        | [OpenAI](https://trigger.dev/docs/integrations/apis/openai)                                                                 | Trigger.dev                             | 🛠️     |
 | AI changelog generator                                                                                           | Generates a changelog from your GitHub commits using OpenAI                                                                                 | [OpenAI](https://trigger.dev/docs/integrations/apis/openai) [GitHub](https://trigger.dev/docs/integrations/apis/github)     | Trigger.dev                             | 🛠️     |

docs/integrations/apis/airtable-tasks.mdx

@@ -15,9 +15,8 @@ Gets multiple records for a table. You can filter. [Official Airtable Docs](http
 
 ```ts example.ts
 // Gets multiple records from the table.
-// Here we only get the Status fields (columns)
 await table.getRecords("multiple records", {
-  fields: ["Status"],
+  fields: ["Status"], // Here we only get the Status fields (columns)
 });
 ```
 
@@ -39,7 +38,6 @@ Create one or more records in a table. [Official Airtable Docs](https://airtable
 await table.createRecords("create records", [
   {
     // Define the new record to be created
-    // with field values for "Launch goals" and "Status."
     fields: {
       "Launch goals": "Created from Trigger.dev",
       Status: "In progress",
@@ -80,16 +78,22 @@ await table.deleteRecords(
 );
 ```
 
-### `runTask`
-
-Do anything that’s possible with the official Airtable Node SDK.
-
 ## Example Usage
 
-You can optionally add the types for your Table – this gives you nice type inference and errors when writing your code.
+In this demo we will use the Airtable to get records from a table, create a new record, update it and then delete it.
 
 ```ts
-// This is the type definition for my Table
+// Importing necessary modules and packages.
+import { TriggerClient, eventTrigger } from "@trigger.dev/sdk";
+import { Airtable } from "@trigger.dev/airtable";
+import { z } from "zod";
+
+// Creating an instance of the Airtable client with personal access token obtained from an environment variable.
+const airtable = new Airtable({
+  id: "airtable",
+  apiKey: process.env.AIRTABLE_PERSONAL_ACCESS_TOKEN!,
+});
+
 type LaunchGoalsAndOkRs = {
   "Launch goals"?: string;
   DRI?: Collaborator;

docs/integrations/apis/airtable.mdx

@@ -7,20 +7,17 @@ sidebarTitle: Overview & authentication
 
 Our Airtable integration allows you to easily connect to the Airtable API and perform tasks such as creating / updating / deleting single or multiple records in your tables.
 
-For examples of some of the things you can do with it, check out our Jobs Showcase:
-
 <Card
   title="Jobs Showcase - Airtable"
   icon="rocket"
-  href="https://trigger.dev/showcase?tags=&integrations=airtable"
+  href="https://trigger.dev/showcase?tags=&apis=airtable"
 >
   Check out pre-built Airtable jobs in our showcase.
 </Card>
 
 ## Installation
 
-To get started with the Airtable integration on Trigger.dev, you need to install the `@trigger.dev/airtable` package.
-You can do this using npm, pnpm, or yarn:
+To get started with our Airtable integration, you need to install the `@trigger.dev/airtable` packages. You can do this using `npm`, `pnpm`, or `yarn`:
 
 <CodeGroup>
 
@@ -44,7 +41,7 @@ To use the Airtable API with Trigger.dev, you can either use OAuth or a Personal
 
 ### OAuth
 
-To use OAuth you can connect to Airtable via the Trigger.dev [web app](https://cloud.trigger.dev). Click 'Integrations' in the side panel of any project, and configure Airtable with the ID you want to use in your job and the required [scopes](https://airtable.com/developers/web/api/scopes).
+To use OAuth you can connect to Airtable via the Trigger.dev [web app](https://cloud.trigger.dev). Click 'Integrations' in the side panel of any project, configure Airtable with a unique ID and the required [scopes](https://airtable.com/developers/web/api/scopes).
 
 ```ts
 import { Airtable } from "@trigger.dev/airtable";
@@ -57,7 +54,7 @@ const airtable = new Airtable({
 
 ### Personal Access Token
 
-You can create an Airtable Personal Access Token [here on their developer site](https://airtable.com/create/tokens).
+You can create an Airtable Personal Access Token [here](https://airtable.com/create/tokens).
 
 ```ts
 import { Airtable } from "@trigger.dev/airtable";
@@ -73,8 +70,6 @@ const airtable = new Airtable({
 
 Once you have set up a Airtable client, you can use it to create tasks.
 
-<CardGroup>
-  <Card title="Tasks" icon="sparkles" href="/integrations/apis/airtable-tasks">
-    Perform tasks such as creating / updating / deleting single or multiple records in table.
-  </Card>
-</CardGroup>
+<Card title="Tasks" icon="sparkles" href="/integrations/apis/airtable-tasks">
+  Perform tasks such as creating / updating / deleting single or multiple records in table.
+</Card>

docs/integrations/apis/github.mdx

@@ -5,22 +5,20 @@ sidebarTitle: Overview & authentication
 
 ## Overview
 
-Our GitHub integration allows you to create triggers and tasks that interact with GitHub.
-
-Trigger jobs when events happen, such as when a new issue is added to a repo, a commit is pushed, or a pull request is opened. You can also use the integration to perform tasks such as creating issues, getting information about a repo, adding comments, and much more.
-
-For examples of some of the things you can do with it, check out our Jobs Showcase:
+Our GitHub integration allows you to create triggers and tasks that interact with GitHub. Trigger jobs when events happen, like when a new issue is added to a repo, or when a pull request is opened, etc. You can also perform tasks like creating issues, getting information about a repo, adding comments, and much more.
 
 <Card
   title="Jobs Showcase - GitHub"
   icon="rocket"
-  href="https://trigger.dev/showcase?tags=&integrations=github"
+  href="https://trigger.dev/showcase?tags=&apis=github"
 >
   Check out pre-built GitHub jobs in our showcase.
 </Card>
 
 ## Installing the GitHub packages
 
+To get started with our GitHub integration, you need to install the `@trigger.dev/github` packages. You can do this using `npm`, `pnpm`, or `yarn`:
+
 <CodeGroup>
 
 ```bash npm

docs/integrations/apis/openai-tasks.mdx

@@ -1,643 +0,0 @@
----
-title: OpenAI tasks
-sidebarTitle: Tasks
----
-
-Tasks are executed after the job is triggered and are the main building blocks of a job. You can string together as many tasks as you want.
-
----
-
-## All tasks
-
-### `createCompletion`
-
-Generates text completions as per given prompt. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // This code demonstrates using OpenAI's text completion with the "davinci" model.
-    // It generates text based on the given prompt.
-    await io.openai.createCompletion("completion", {
-      model: "davinci",
-      prompt: "Once upon a time",
-    });
-},
-```
-
-### `backgroundCreateCompletion`
-
-Generates text completions in the background. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // This code showcases background text completion using the "gpt-3.5-turbo" model.
-    // It generates text based on the provided programming task and logs the result.
-    const programmingTask = `Create a function that checks if a string is a palindrome.`;
-
-    const response = await io.openai.backgroundCreateCompletion("background-completion", {
-      model: "gpt-3.5-turbo",
-      prompt: `Coding task: ${programmingTask}\n\n`,
-    });
-
-    await io.logger.info("codeSnippet", response.choices[0]?.text);
-},
-```
-
-You can also pass an optional third parameter to `backgroundCreateCompletion` to specify OpenAI request options:
-
-```ts requestOptions.ts
-run: async (payload, io, ctx) => {
-    // This code showcases background text completion using the "gpt-3.5-turbo" model.
-    // It generates text based on the provided programming task and logs the result.
-    const programmingTask = `Create a function that checks if a string is a palindrome.`;
-
-    const response = await io.openai.backgroundCreateCompletion("background-completion", {
-      model: "gpt-3.5-turbo",
-      prompt: `Coding task: ${programmingTask}\n\n`,
-    }, {
-      headers: {
-        "User-Agent": "my-user-agent"
-      }
-    });
-
-    await io.logger.info("codeSnippet", response.choices[0]?.text);
-},
-```
-
-This task is implemented using [io.backgroundFetch()](/sdk/io/backgroundfetch) and so you can also pass a 4th parameter customizing the retry and timeout options:
-
-```ts fetchOptions.ts
-run: async (payload, io, ctx) => {
-    // This code showcases background text completion using the "gpt-3.5-turbo" model.
-    // It generates text based on the provided programming task and logs the result.
-    const programmingTask = `Create a function that checks if a string is a palindrome.`;
-
-    const response = await io.openai.backgroundCreateCompletion("background-completion", {
-      model: "gpt-3.5-turbo",
-      prompt: `Coding task: ${programmingTask}\n\n`,
-    }, {
-      headers: {
-        "User-Agent": "my-user-agent"
-      }
-    }, {
-      timeout: {
-        durationInMs: 10000,
-        retry: {
-          limit: 3,
-          minTimeoutInMs: 1000,
-          factor: 2,
-        }
-      }
-    });
-
-    await io.logger.info("codeSnippet", response.choices[0]?.text);
-},
-```
-
-### `createChatCompletion`
-
-Generates text completions in a conversational context. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat/create)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // This code demonstrates chat completion with the "gpt-3.5-turbo" model.
-    // It simulates a conversation by providing messages and receiving a chat response.
-    await io.openai.createChatCompletion("chat-completion", {
-      model: "gpt-3.5-turbo",
-      messages: [
-        {
-          role: "user",
-          content: "Create a good programming joke about background jobs",
-        },
-      ],
-    });
-},
-```
-
-### `backgroundCreateChatCompletion`
-
-Generates text completions in a conversational context in the background. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat/object)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // This code showcases background chat completion using the "gpt-3.5-turbo" model.
-    // It simulates a conversation with a user message and logs the response choices.
-    const response = await io.openai.backgroundCreateChatCompletion("background-chat-completion", {
-      model: "gpt-3.5-turbo",
-      messages: [
-        {
-          role: "user",
-          content: "Create a good programming joke about background jobs",
-        },
-      ],
-    });
-
-    await io.logger.info("choices", response.choices);
-},
-```
-
-You can also use the more "fluent" pattern used by the OpenAI SDK:
-
-```ts fluent.ts
-run: async (payload, io, ctx) => {
-    // This code showcases background chat completion using the "gpt-3.5-turbo" model.
-    // It simulates a conversation with a user message and logs the response choices.
-    const response = await io.openai.chat.completions.backgroundCreate("background-chat-completion", {
-      model: "gpt-3.5-turbo",
-      messages: [
-        {
-          role: "user",
-          content: "Create a good programming joke about background jobs",
-        },
-      ],
-    });
-
-    await io.logger.info("choices", response.choices);
-},
-```
-
-Additionally, you can pass an optional third parameter to specify OpenAI request options:
-
-```ts requestOptions.ts
-run: async (payload, io, ctx) => {
-    // This code showcases background text completion using the "gpt-3.5-turbo" model.
-    // It generates text based on the provided programming task and logs the result.
-    const programmingTask = `Create a function that checks if a string is a palindrome.`;
-
-    const response = await io.openai.chat.completions.backgroundCreate("background-completion", {
-      model: "gpt-3.5-turbo",
-      messages: [
-        {
-          role: "user",
-          content: "Create a good programming joke about background jobs",
-        },
-      ],
-    }, {
-      headers: {
-        "User-Agent": "my-user-agent"
-      }
-    });
-
-    await io.logger.info("choices", response.choices);
-},
-```
-
-This task is implemented using [io.backgroundFetch()](/sdk/io/backgroundfetch) and so you can also pass a 4th parameter customizing the retry and timeout options:
-
-```ts fetchOptions.ts
-run: async (payload, io, ctx) => {
-    // This code showcases background text completion using the "gpt-3.5-turbo" model.
-    // It generates text based on the provided programming task and logs the result.
-    const programmingTask = `Create a function that checks if a string is a palindrome.`;
-
-    const response = await io.openai.chat.completions.backgroundCreate("background-completion", {
-      model: "gpt-3.5-turbo",
-      messages: [
-        {
-          role: "user",
-          content: "Create a good programming joke about background jobs",
-        },
-      ],
-    }, {
-      headers: {
-        "User-Agent": "my-user-agent"
-      }
-    }, {
-      timeout: {
-        durationInMs: 10000,
-        retry: {
-          limit: 3,
-          minTimeoutInMs: 1000,
-          factor: 2,
-        }
-      }
-    });
-
-    await io.logger.info("choices", response.choices);
-},
-```
-
-### `retrieveModel`
-
-Retrieves a specific model by ID. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/models/retrieve)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // In this code snippet, we retrieve detailed information about a specific OpenAI model.
-
-    // Specify the ID of the model you want to retrieve. Replace 'your_model_id' with the actual model ID.
-    const modelIdToRetrieve = "your_model_id";
-
-    try {
-        // Retrieve the model information using the OpenAI API
-        const retrievedModel = await io.openai.retrieveModel("get-model", {
-            model: modelIdToRetrieve,
-        });
-
-        // Log the detailed model information
-        await io.logger.info("retrievedModel", retrievedModel);
-    } catch (error) {
-        // Handle errors, such as if the model with the provided ID does not exist.
-        await io.logger.error("Error retrieving model:", error.message);
-    }
-},
-```
-
-### `listModels`
-
-Lists the available models. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/models/list)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // This code lists available models without retrieving detailed information.
-    const models = await io.openai.listModels("list-models");
-},
-```
-
-### `createEdit`
-
-Edits a given text prompt. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/edits/create)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // This code snippet demonstrates using the OpenAI API to create an edit task.
-
-    // Specify the task parameters:
-    const editTaskParams = {
-        model: "text-davinci-edit-001", // Replace with the desired model
-        input: "Thsi is ridddled with erors", // Replace with the input text
-        instruction: "Fix the spelling errors", // Replace with the editing instruction
-    };
-
-    try {
-        // Create an edit task using the OpenAI API
-        const editResponse = await io.openai.createEdit("edit", editTaskParams);
-
-        // Log the response
-        await io.logger.info("editResponse", editResponse);
-    } catch (error) {
-        // Handle any potential errors that may occur during the API request.
-        await io.logger.error("Error creating edit task:", error.message);
-    }
-},
-```
-
-### `createImage`
-
-Generates images from textual descriptions. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/images/create)
-
-```ts example.ts
-  run: async (payload, io, ctx) => {
-    const imageResults = await io.openai.createImage("image", {
-      prompt: "A hedgehog wearing a party hat",
-      n: 2,
-      size: "256x256",
-      response_format: "url",
-    });
-```
-
-### `createImageEdit`
-
-Creates an edited or extended image given an original image and a prompt. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/images/createEdit)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the parameters for the image edit
-  const imageEditParams = {
-    style: "data:image/png;base64,base64_encoded_style_image",
-    content: "data:image/png;base64,base64_encoded_content_image",
-  };
-
-  // Create the image edit using the OpenAI API
-  const imageEditResponse = await io.openai.createImageEdit(imageEditParams);
-
-  // Log the response
-  await io.logger.info("imageEditResponse", imageEditResponse);
-},
-```
-
-### `createImageVariation`
-
-Creates a variation of a given image. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/images/createVariation)
-
-```ts example.ts
-    run: async (payload, io, ctx) => {
-        // Specify the parameters for creating an image variation
-        const imageVariationParams = {
-        image: "data:image/png;base64,base64_encoded_image",
-        variation: "brightness(1.2) contrast(0.8) rotate(45deg)",
-        };
-
-        // Create the image variation using the OpenAI API
-        const imageVariationResponse = await io.openai.createImageVariation(imageVariationParams);
-
-        // Log the response
-        await io.logger.info("imageVariationResponse", imageVariationResponse);
-    },
-```
-
-### `createEmbedding`
-
-Generates embeddings for a given text. [Official OpenAI Docs](hhttps://platform.openai.com/docs/api-reference/embeddings/object)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-    // This code snippet demonstrates using the OpenAI API to create a text embedding.
-
-    // Specify the task parameters:
-    const embeddingTaskParams = {
-        model: "text-embedding-ada-002", // Replace with the desired model
-        input: "The food was delicious and the waiter...", // Replace with the input text
-    };
-
-    try {
-        // Create a text embedding using the OpenAI API
-        const embeddingResponse = await io.openai.createEmbedding("embedding", embeddingTaskParams);
-
-        // Log the response
-        await io.logger.info("embeddingResponse", embeddingResponse);
-    } catch (error) {
-        // Handle any potential errors that may occur during the API request.
-        await io.logger.error("Error creating text embedding:", error.message);
-    }
-},
-```
-
-### `createFile`
-
-Uploads a file to the OpenAI API. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/files/object)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the parameters for creating a file
-  const fileParams = {
-    name: "example.txt",
-    content: "This is the content of the file.",
-  };
-
-  // Create the file using the OpenAI API
-  const fileResponse = await io.openai.createFile(fileParams);
-
-  // Log the response
-  await io.logger.info("fileResponse", fileResponse);
-},
-```
-
-### `listFiles`
-
-Lists the uploaded files. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/files/list)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // List the files available in your OpenAI account
-  const fileListResponse = await io.openai.listFiles();
-
-  // Log the list of files
-  await io.logger.info("fileListResponse", fileListResponse);
-},
-```
-
-### `createFineTuneFile`
-
-Uploads a file for fine-tuning a model. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the parameters for creating a fine-tune file
-  const fineTuneFileParams = {
-    model: "text-davinci-002",
-    prompt: "Translate English to French: 'Hello, world.'",
-    language: "en",
-    description: "Fine-tune file for translation task",
-  };
-
-  // Create the fine-tune file using the OpenAI API
-  const fineTuneFileResponse = await io.openai.createFineTuneFile(fineTuneFileParams);
-
-  // Log the response
-  await io.logger.info("fineTuneFileResponse", fineTuneFileResponse);
-},
-```
-
-### `createFineTune`
-
-Fine-tunes a model on a given task. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/create)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the parameters for creating a fine-tune task
-  const fineTuneParams = {
-    model: "text-davinci-002",
-    dataset: "your_dataset_id",
-    description: "Fine-tune task for custom dataset",
-  };
-
-  // Create the fine-tune task using the OpenAI API
-  const fineTuneResponse = await io.openai.createFineTune(fineTuneParams);
-
-  // Log the response
-  await io.logger.info("fineTuneResponse", fineTuneResponse);
-},
-```
-
-### `listFineTunes`
-
-Lists the available fine-tunes. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/list)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // List the fine-tunes available in your OpenAI account
-  const fineTunesListResponse = await io.openai.listFineTunes();
-
-  // Log the list of fine-tunes
-  await io.logger.info("fineTunesListResponse", fineTunesListResponse);
-},
-```
-
-### `retrieveFineTune`
-
-Retrieves a specific fine-tune by ID. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/retrieve)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the ID of the fine-tune you want to retrieve
-  const fineTuneId = "your_fine_tune_id"; // Replace with the actual fine-tune ID
-
-  // Retrieve the fine-tune using the OpenAI API
-  const retrievedFineTune = await io.openai.retrieveFineTune(fineTuneId);
-
-  // Log the retrieved fine-tune
-  await io.logger.info("retrievedFineTune", retrievedFineTune);
-},
-```
-
-### `cancelFineTune`
-
-Cancels a specific fine-tune by ID. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/cancel)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the ID of the fine-tune you want to cancel
-  const fineTuneIdToCancel = "your_fine_tune_id"; // Replace with the actual fine-tune ID
-
-  // Cancel the specified fine-tune using the OpenAI API
-  const cancellationResponse = await io.openai.cancelFineTune(fineTuneIdToCancel);
-
-  // Log the cancellation response
-  await io.logger.info("cancellationResponse", cancellationResponse);
-},
-```
-
-### `createFineTuningJob`
-
-Creates a job that fine-tunes a specified model from a given dataset. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/create)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the parameters for creating a fine-tuning job
-  const fineTuningJobParams = {
-    fineTuneId: "your_fine_tune_id", // Replace with the actual fine-tune ID
-    datasetId: "your_dataset_id", // Replace with the ID of your dataset
-    model: "text-davinci-002", // Replace with the model for fine-tuning
-    n_examples: 100, // Replace with the number of examples
-  };
-
-  // Create the fine-tuning job using the OpenAI API
-  const fineTuningJobResponse = await io.openai.createFineTuningJob(fineTuningJobParams);
-
-  // Log the response
-  await io.logger.info("fineTuningJobResponse", fineTuningJobResponse);
-},
-```
-
-### `retrieveFineTuningJob`
-
-Get info about a fine-tuning job. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/retrieve)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the ID of the fine-tuning job you want to retrieve
-  const fineTuningJobId = "your_fine_tuning_job_id"; // Replace with the actual job ID
-
-  // Retrieve the fine-tuning job using the OpenAI API
-  const retrievedJob = await io.openai.retrieveFineTuningJob(fineTuningJobId);
-
-  // Log the retrieved job
-  await io.logger.info("retrievedJob", retrievedJob);
-},
-```
-
-### `cancelFineTuningJob`
-
-Cancel a fine-tuning job. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/cancel)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the ID of the fine-tuning job you want to cancel
-  const fineTuningJobIdToCancel = "your_fine_tuning_job_id"; // Replace with the actual job ID
-
-  // Cancel the specified fine-tuning job using the OpenAI API
-  const cancellationResponse = await io.openai.cancelFineTuningJob(fineTuningJobIdToCancel);
-
-  // Log the cancellation response
-  await io.logger.info("cancellationResponse", cancellationResponse);
-},
-```
-
-### `listFineTuningJobEvents`
-
-List events for a fine-tuning job. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/list-events)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // Specify the ID of the fine-tuning job for which you want to list events
-  const fineTuningJobId = "your_fine_tuning_job_id"; // Replace with the actual job ID
-
-  // List events for the specified fine-tuning job using the OpenAI API
-  const eventsListResponse = await io.openai.listFineTuningJobEvents(fineTuningJobId);
-
-  // Log the list of events
-  await io.logger.info("eventsListResponse", eventsListResponse);
-},
-```
-
-### `listFineTuningJobs`
-
-List fine tuning jobs. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/list)
-
-```ts example.ts
-run: async (payload, io, ctx) => {
-  // List the fine-tuning jobs available in your OpenAI account
-  const jobsListResponse = await io.openai.listFineTuningJobs();
-
-  // Log the list of fine-tuning jobs
-  await io.logger.info("jobsListResponse", jobsListResponse);
-},
-```
-
-## Example usage
-
-In this example we'll create a task that generates a random joke using OpenAI GPT 3.5 .
-
-```ts example.ts
-import { TriggerClient, eventTrigger } from "@trigger.dev/sdk";
-import { OpenAI } from "@trigger.dev/openai";
-import { z } from "zod";
-
-// Initialize a TriggerClient with the ID "jobs-showcase"
-const client = new TriggerClient({ id: "jobs-showcase" });
-
-// Create an instance of the OpenAI client and provide the OpenAI API key from environment variables
-const openai = new OpenAI({
-  id: "openai",
-  apiKey: process.env.OPENAI_API_KEY!, // Replace with your actual OpenAI API key
-});
-
-// Define a job that uses OpenAI GPT-3.5 Turbo to tell jokes
-client.defineJob({
-  id: "openai-tell-me-a-joke",
-  name: "OpenAI: tell me a joke",
-  version: "1.0.0",
-  trigger: eventTrigger({
-    name: "openai.tasks", // Define the trigger event name
-    schema: z.object({
-      jokePrompt: z.string(), // Expect a joke prompt as input
-    }),
-  }),
-  integrations: {
-    openai, // Use the OpenAI integration for this job
-  },
-  run: async (payload, io, ctx) => {
-    // Retrieve information about the GPT-3.5 Turbo model
-    await io.openai.retrieveModel("get-model", {
-      model: "gpt-3.5-turbo",
-    });
-
-    // List available models (optional, for reference)
-    const models = await io.openai.listModels("list-models");
-
-    // Generate a joke in the background using the chat conversation format
-    const jokeResult = await io.openai.backgroundCreateChatCompletion(
-      "background-chat-completion",
-      {
-        model: "gpt-3.5-turbo",
-        messages: [
-          {
-            role: "user",
-            content: payload.jokePrompt, // User-provided joke prompt
-          },
-        ],
-      }
-    );
-
-    // Return the generated joke as the result
-    return {
-      joke: jokeResult.choices[0]?.message?.content,
-    };
-  },
-});
-
-// These lines are specific to the Express framework and can be removed if not needed
-import { createExpressServer } from "@trigger.dev/express";
-createExpressServer(client);
-```

docs/integrations/apis/openai.mdx

@@ -10,7 +10,7 @@ Our OpenAI integration allows you to easily perform AI-powered tasks, such as su
 <Card
   title="Jobs Showcase - OpenAI"
   icon="rocket"
-  href="https://trigger.dev/showcase?tags=&integrations=openai"
+  href="https://trigger.dev/showcase?tags=&apis=openai"
 >
   Check out pre-built OpenAI jobs in our showcase.
 </Card>
@@ -49,11 +49,71 @@ const openai = new OpenAI({
 
 ## Tasks
 
-Once you have set up a OpenAI client, you can use it to create tasks.
+Once you have set up a OpenAI client, you can add it to your job and start using the provided tasks:
+
+```ts
+client.defineJob({
+  id: "openai-job",
+  name: "OpenAI Job",
+  version: "1.0.0",
+  trigger: invokeTrigger(),
+  integrations: {
+    openai, // Add the OpenAI client as an integration
+  },
+  run: async (payload, io, ctx) => {
+    // Now you can access it through the io object
+    const completion = await io.openai.chat.completions.create("completion", {
+      model: "gpt-3.5-turbo",
+      messages: [
+        {
+          role: "user",
+          content: "Create a good programming joke about background jobs",
+        },
+      ],
+    });
+  },
+});
+```
+
+As you can see above, we've replicated the API of the [OpenAI TypeScript SDK](https://github.com/openai/openai-node), with a crucial difference of adding the [Task Cache Key](https://trigger.dev/docs/documentation/concepts/tasks#task-cache-keys) as the first parameter.
+
+We've also added a few convenience methods to make it easier to work with the OpenAI API, especially in a serverless environment. For example, you can run a Chat Completion task in the background with [backgroundCreate()](/integrations/apis/openai/chat#completions-backgroundcreate):
+
+```ts
+const completion = await io.openai.chat.completions.backgroundCreate("completion", {
+  model: "gpt-3.5-turbo",
+  messages: [
+    {
+      role: "user",
+      content: "Create a good programming joke about background jobs",
+    },
+  ],
+});
+```
+
+See our full task reference below:
 
 <CardGroup>
-  <Card title="Tasks" icon="sparkles" href="/integrations/apis/openai-tasks">
-    Perform different AI-powered tasks using OpenAI.
+  <Card title="Chat Completions" icon="sparkles" href="/integrations/apis/openai/chat">
+    Given a list of messages comprising a conversation, the model will return a response
+  </Card>
+  <Card title="Assistants (Beta)" icon="arrows-spin" href="/integrations/apis/openai/assistants">
+    Build assistants that can call models and use tools to perform tasks
+  </Card>
+  <Card title="Files" icon="file" href="/integrations/apis/openai/files">
+    Upload files to use with assistants and fine-tuning
+  </Card>
+  <Card title="Images" icon="image" href="/integrations/apis/openai/images">
+    Given a prompt and/or an input image, the model will generate a new image
+  </Card>
+  <Card title="Fine Tuning Jobs" icon="vial" href="/integrations/apis/openai/fine-tunes">
+    Manage fine-tuning jobs to tailor a model to your specific training data
+  </Card>
+  <Card title="Models" icon="server" href="/integrations/apis/openai/models">
+    List and describe the various models available in the API
+  </Card>
+  <Card title="Completions (Legacy)" icon="scroll" href="/integrations/apis/openai/completions">
+    Given a prompt, the model will return one or more predicted completions.
   </Card>
 </CardGroup>
 

docs/integrations/apis/openai/assistants.mdx

@@ -0,0 +1,287 @@
+---
+title: Assistant Tasks
+sidebarTitle: Assitants (Beta)
+---
+
+<Note>
+  This feature is currently marked as a "Beta" by OpenAI. Make sure to check our their [How
+  Assistants Work](https://platform.openai.com/docs/assistants/how-it-works) and [Assistants
+  Overview](https://platform.openai.com/docs/assistants/overview) guides.
+</Note>
+
+## Assistants
+
+Build assistants that can call models and use tools to perform tasks. [Official OpenAI docs](https://platform.openai.com/docs/api-reference/assistants)
+
+### `create()`
+
+Create an assistant with a model and instructions. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/assistants/createAssistant)
+
+```ts example.ts
+const file = await io.openai.files.createAndWaitForProcessing("upload-file", {
+  purpose: "assistants",
+  file: fs.createReadStream("./fixtures/mydata.csv"),
+});
+
+const assistant = await io.openai.beta.assistants.create("create-assistant", {
+  name: "Data visualizer",
+  description:
+    "You are great at creating beautiful data visualizations. You analyze data present in .csv files, understand trends, and come up with data visualizations relevant to those trends. You also share a brief text summary of the trends observed.",
+  model: "gpt-4-1106-preview",
+  tools: [{ type: "code_interpreter" }],
+  file_ids: [file.id],
+});
+```
+
+## Threads
+
+Create threads that assistants can interact with. [Official OpenAI docs](https://platform.openai.com/docs/api-reference/threads/createThread)
+
+### `create()`
+
+Create a thread. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/assistants/createAssistant)
+
+```ts example.ts
+const thread = await io.openai.beta.threads.create("create-thread", {
+  messages: [
+    {
+      role: "user",
+      content: "Create 3 data visualizations based on the trends in this file.",
+      file_ids: [fileId],
+    },
+  ],
+});
+```
+
+### `createAndRun()`
+
+Create a thread and run it in one task.
+
+```ts example.ts
+const run = await io.openai.beta.threads.createAndRun("create-and-run-thread", {
+  assistant_id: "asst_abc123",
+  thread: {
+    messages: [
+      {
+        role: "user",
+        content: "Create 3 data visualizations based on the trends in this file.",
+        file_ids: [fileId],
+      },
+    ],
+  },
+});
+```
+
+### `createAndRunUntilCompletion()`
+
+Create a thread and runs it in one task, and only returns when the run is completed by polling in the background using [io.backgroundPoll()](/sdk/io/background-poll).
+
+```ts example.ts
+const run = await io.openai.beta.threads.createAndRunUntilCompletion("create-thread", {
+  assistant_id: "asst_abc123",
+  thread: {
+    messages: [
+      {
+        role: "user",
+        content: "Create 3 data visualizations based on the trends in this file.",
+        file_ids: [fileId],
+      },
+    ],
+  },
+});
+
+if (run.status !== "completed") {
+  throw new Error(`Run finished with status ${run.status}: ${JSON.stringify(run.last_error)}`);
+}
+
+// List all messages in the thread
+const messages = await io.openai.beta.threads.messages.list("list-messages", run.thread_id);
+```
+
+### `retrieve()`
+
+Retrieves a thread. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/threads/getThread)
+
+```ts example.ts
+const thread = await io.openai.beta.threads.retrieve("get-thread", "thread_abc123");
+```
+
+### `update()`
+
+Modifies a thread. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/threads/modifyThread)
+
+```ts example.ts
+await io.openai.beta.threads.update("update-thread", "thread_abc123", {
+  metadata: {
+    foo: "bar",
+  },
+});
+```
+
+### `del()`
+
+Deletes a thread. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/threads/deleteThread)
+
+```ts example.ts
+const deletedThread = await io.openai.beta.threads.del("update-thread", "thread_abc123");
+```
+
+## Messages
+
+Create messages within threads. [Official OpenAI docs](https://platform.openai.com/docs/api-reference/messages)
+
+### `list()`
+
+List all messages in a thread.
+
+```ts example.ts
+const messages = await io.openai.beta.threads.messages.list("list-messages", "thread_abc123");
+```
+
+### `create()`
+
+Create a message. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/messages/createMessage)
+
+```ts example.ts
+const thread = await io.openai.beta.threads.create("get-thread");
+
+const message = await io.openai.beta.threads.messages.create("create-message", thread.id, {
+  role: "user",
+  content: "Create 3 data visualizations based on the trends in this file.",
+  file_ids: [fileId],
+});
+```
+
+### `retrieve()`
+
+Retrieve a message. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/messages/getMessage)
+
+```ts example.ts
+const message = await io.openai.beta.threads.messages.retrieve(
+  "get-message",
+  "thread_abc123",
+  "message_abc123"
+);
+```
+
+### `update()`
+
+Update a message. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/messages/modifyMessage)
+
+```ts example.ts
+await io.openai.beta.threads.messages.update("update-message", thread.id, message.id, {
+  metadata: {
+    foo: "bar",
+  },
+});
+```
+
+## Runs
+
+Represents an execution run on a thread. [Official OpenAI docs](https://platform.openai.com/docs/api-reference/runs)
+
+### `list()`
+
+List all runs belonging to a thread.
+
+```ts example.ts
+const runs = await io.openai.beta.threads.runs.list("list-runs", "thread_abc123");
+```
+
+### `create()`
+
+Create a run. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/runs/createRun)
+
+```ts example.ts
+const run = await io.openai.beta.threads.runs.create("create-run", "thread_abc123", {
+  assistant_id: payload.id,
+});
+```
+
+### `createAndWaitForCompletion()`
+
+Create a run and only return when the run is completed by polling in the background using [io.backgroundPoll()](/sdk/io/background-poll).
+
+```ts example.ts
+const run = await io.openai.beta.threads.runs.createAndWaitForCompletion(
+  "create-run",
+  "thread_abc123",
+  {
+    assistant_id: payload.id,
+  }
+);
+
+if (run.status !== "completed") {
+  throw new Error(`Run finished with status ${run.status}: ${JSON.stringify(run.last_error)}`);
+}
+
+const messages = await io.openai.beta.threads.messages.list("list-messages", run.thread_id);
+```
+
+### `waitForCompletion()`
+
+Wait for a run to complete by polling in the background using [io.backgroundPoll()](/sdk/io/background-poll).
+
+```ts example.ts
+const run = await io.openai.beta.threads.runs.create("create-run", "thread_abc123", {
+  assistant_id: payload.id,
+});
+
+const completedRun = await io.openai.beta.threads.runs.waitForCompletion(
+  "wait-for-completion",
+  "thread_abc123",
+  run.id
+);
+
+if (completedRun.status !== "completed") {
+  throw new Error(
+    `Run finished with status ${completedRun.status}: ${JSON.stringify(completedRun.last_error)}`
+  );
+}
+
+const messages = await io.openai.beta.threads.messages.list("list-messages", run.thread_id);
+```
+
+### `retrieve()`
+
+Retrieve a run. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/runs/getRun)
+
+```ts example.ts
+const run = await io.openai.beta.threads.runs.retrieve("get-run", "thread_abc123", "run_abc123");
+```
+
+### `cancel()`
+
+Cancels a run that is `in_progress`. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/runs/cancelRun)
+
+```ts example.ts
+const run = await io.openai.beta.threads.runs.cancel("cancel-run", "thread_abc123", "run_abc123");
+```
+
+### `submitToolOutputs()`
+
+When a run has the `status: "requires_action"` and `required_action.type` is `submit_tool_outputs`, this endpoint can be used to submit the outputs from the tool calls once they're all completed. All outputs must be submitted in a single request. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/runs/submitToolOutputs)
+
+```ts example.ts
+const run = await io.openai.beta.threads.runs.submitToolOutputs(
+  "submit-tool-outputs",
+  "thread_abc123",
+  "run_abc123",
+  {
+    tool_outputs: [
+      {
+        tool_call_id: "tool_run_abc123",
+        output: "This is the output of the tool call.",
+      },
+    ],
+  }
+);
+```
+
+### `list()`
+
+Returns all runs belonging to a thread. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/runs/listRuns)
+
+```ts example.ts
+const runs = await io.openai.beta.threads.runs.list("list-runs", "thread_abc123");
+```

docs/integrations/apis/openai/chat.mdx

@@ -0,0 +1,38 @@
+---
+title: Chat Completion Tasks
+sidebarTitle: Chat Completions
+---
+
+Given a list of messages comprising a conversation, the model will return a response. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat)
+
+### `completions.create()`
+
+Creates a model response for the given chat conversation. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat/create)
+
+```ts example.ts
+await io.openai.chat.completions.create("chat-completion", {
+  model: "gpt-3.5-turbo",
+  messages: [
+    {
+      role: "user",
+      content: "Create a good programming joke about background jobs",
+    },
+  ],
+});
+```
+
+### `completions.backgroundCreate()`
+
+Creates a model response for the given chat conversation, but runs the request in the background using [io.backgroundFetch()](/sdk/io/backgroundfetch)
+
+```ts example.ts
+await io.openai.chat.completions.create("chat-completion", {
+  model: "gpt-3.5-turbo",
+  messages: [
+    {
+      role: "user",
+      content: "Create a good programming joke about background jobs",
+    },
+  ],
+});
+```

docs/integrations/apis/openai/completions.mdx

@@ -0,0 +1,19 @@
+---
+title: Completion Tasks
+sidebarTitle: Completions (Legacy)
+---
+
+Given a prompt, the model will return one or more predicted completions, and can also return the probabilities of alternative tokens at each position. We recommend most users use the Chat Completions API. [Learn more](https://platform.openai.com/docs/deprecations/2023-07-06-gpt-and-embeddings)
+
+### `create()`
+
+<Warning>This is a legacy API</Warning>
+
+Creates a completion for the provided prompt and parameters. [Official OpenAI docs](https://platform.openai.com/docs/api-reference/completions/create)
+
+```ts example.ts
+const completion = await io.openai.completions.create("completion", {
+  model: "text-davinci-003",
+  prompt: "Create a good programming joke about Tasks",
+});
+```

docs/integrations/apis/openai/files.mdx

@@ -0,0 +1,62 @@
+---
+title: File Tasks
+sidebarTitle: Files
+---
+
+Files are used to upload documents that can be used with features like [Assistants](https://platform.openai.com/docs/api-reference/assistants) and [Fine-tuning](https://platform.openai.com/docs/api-reference/fine-tuning). [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/files)
+
+### `list()`
+
+Returns a list of files that belong to the user's organization. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/files/list)
+
+```ts example.ts
+await io.openai.files.list("list-files");
+await io.openai.files.list("list-files", { purpose: "assistants" }); // gets only assistant files
+```
+
+### `create()`
+
+Upload a file that can be used across various endpoints/features. The size of all the files uploaded by one organization can be up to 100 GB.
+
+The size of individual files for can be a maximum of `512MB`. See the [Assistants Tools guide](https://platform.openai.com/docs/assistants/tools) to learn more about the types of files supported. The Fine-tuning API only supports `.jsonl` files.
+
+[Official OpenAI Docs](https://platform.openai.com/docs/api-reference/files/create)
+
+```ts example.ts
+const file = await io.openai.files.create("upload-file", {
+  purpose: "assistants",
+  file: fs.createReadStream("./fixtures/mydata.csv"),
+});
+```
+
+### `createAndWaitForProcessing()`
+
+Upload a file and will return when the file is processed by polling in the background using [io.backgroundPoll()](/sdk/io/background-poll).
+
+```ts example.ts
+const file = await io.openai.files.createAndWaitForProcessing("upload-file", {
+  purpose: "assistants",
+  file: fs.createReadStream("./fixtures/mydata.csv"),
+});
+```
+
+### `waitForProcessing()`
+
+Will return when the file is processed by polling in the background using [io.backgroundPoll()](/sdk/io/background-poll).
+
+```ts example.ts
+const file = await io.openai.files.create("upload-file", {
+  purpose: "assistants",
+  file: fs.createReadStream("./fixtures/mydata.csv"),
+});
+
+const processedFile = await io.openai.files.waitForProcessing("wait", file.id);
+```
+
+### `retrieve()`
+
+Returns information about a specific file. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/files/retrieve)
+
+```ts example.ts
+await io.openai.files.retrieve("retrieve-file", "file-id");
+```

docs/integrations/apis/openai/fine-tunes.mdx

@@ -0,0 +1,59 @@
+---
+title: Fine Tuning Tasks
+sidebarTitle: Fine Tunes
+---
+
+Manage fine-tuning jobs to tailor a model to your specific training data. See the related guide [Fine Tuning models](https://platform.openai.com/docs/guides/fine-tuning) and view the [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning).
+
+### `jobs.create()`
+
+Creates a job that fine-tunes a specified model from a given dataset.
+
+Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete.
+
+You must first upload a dataset to the API before creating a fine-tuning job. See our [OpenAI File Tasks](/integrations/apis/openai/files#createandwaitforprocessing) for more information.
+
+```ts example.ts
+const file = await io.openai.files.create("upload-file", {
+  purpose: "fine-tune",
+  file: fs.createReadStream("./mydata.jsonl"),
+});
+
+const fineTuning = await io.openai.fineTuning.jobs.create("fine-tuning", {
+  training_file: file.id,
+  model: "gpt-3.5-turbo",
+  suffix: "my-model",
+});
+```
+
+### `jobs.list()`
+
+List your organization's fine-tuning jobs. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/list)
+
+```ts example.ts
+const fts = await io.openai.fineTuning.jobs.list("list");
+```
+
+### `jobs.retrieve()`
+
+Get info about a fine-tuning job. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/retrieve)
+
+```ts example.ts
+const fineTuning = await io.openai.fineTuning.jobs.retrieve("fine-tuning", "ft_1234");
+```
+
+### `jobs.cancel()`
+
+Immediately cancel a fine-tune job. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/cancel)
+
+```ts example.ts
+const fineTuning = await io.openai.fineTuning.jobs.cancel("fine-tuning", "ft_1234");
+```
+
+### `jobs.listEvents()`
+
+Get status updates for a fine-tuning job. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/fine-tuning/list-events)
+
+```ts example.ts
+const events = await io.openai.fineTuning.jobs.listEvents("fine-tuning", { id: "ft_1234" });
+```

docs/integrations/apis/openai/images.mdx

@@ -0,0 +1,55 @@
+---
+title: Image Tasks
+sidebarTitle: Images
+---
+
+Given a prompt and/or an input image, the model will generate a new image. See the [Image generation guide](https://platform.openai.com/docs/guides/images) and the [Official OpenAI docs](https://platform.openai.com/docs/api-reference/images).
+
+### `create()`
+
+Creates an image given a prompt. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/images/create)
+
+```ts example.ts
+await io.openai.images.create("dalle-3", {
+  model: "dall-e-3",
+  prompt:
+    "I would like to generate an image of an american giraffe riding a bycicle in a suburban neighborhood, into the sunset.",
+});
+```
+
+### `backgroundCreate()`
+
+Creates a an image given a prompt, but runs the request in the background using [io.backgroundFetch()](/sdk/io/backgroundfetch)
+
+```ts example.ts
+await io.openai.images.backgroundCreate("dalle-3", {
+  model: "dall-e-3",
+  prompt:
+    "I would like to generate an image of an american giraffe riding a bycicle in a suburban neighborhood, into the sunset.",
+});
+```
+
+### `edit()`
+
+Creates an edited or extended image given an original image and a prompt. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/images/createEdit)
+
+```ts example.ts
+await io.openai.images.edit("dalle-2", {
+  model: "dall-e-2",
+  image: fs.createReadStream("./giraffe.jpg"),
+  prompt: "A painting of a giraffe in a suburban neighborhood",
+  response_format: "url",
+});
+```
+
+### `createVariation()`
+
+Creates a variation of a given image. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/images/createVariation)
+
+```ts example.ts
+await io.openai.images.createVariation("dalle-3", {
+  model: "dall-e-2",
+  image: fs.createReadStream("./giraffe.jpg"),
+  response_format: "url",
+});
+```

docs/integrations/apis/openai/models.mdx

@@ -0,0 +1,14 @@
+---
+title: Model Tasks
+sidebarTitle: Models
+---
+
+List and describe the various models available in the API. You can refer to the Models documentation to understand what models are available and the differences between them. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/models)
+
+### `list`
+
+Lists the available models. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/models/list)
+
+```ts example.ts
+const models = await io.openai.models.list("list-models");
+```

docs/integrations/apis/plain.mdx

@@ -8,11 +8,10 @@ sidebarTitle: Overview & authentication
 Plain is the customer support tool for technical teams and products.
 It aims to bring engineering and customer service teams together by creating a modern opinionated platform that's fantastic to build with.
 
-
 <Card
   title="Jobs Showcase - Plain"
   icon="rocket"
-  href="https://trigger.dev/showcase?tags=&integrations=plain"
+  href="https://trigger.dev/showcase?tags=&apis=plain"
 >
   Check out pre-built Plain jobs in our showcase.
 </Card>

docs/integrations/apis/resend-tasks.mdx

@@ -0,0 +1,77 @@
+---
+title: Resend tasks
+sidebarTitle: Tasks
+---
+
+Tasks are executed after the job is triggered and are the main building blocks of a job. You can string together as many tasks as you want.
+
+---
+
+## All tasks
+
+### `sendEmail`
+
+Send an email to a recipient with a text payload. [Official Resend Docs](https://resend.com/docs/api-reference/emails/send-email)
+
+```ts example.ts
+  // Using the 'io.resend.sendEmail' method to send an email using Resend.
+  await io.resend.sendEmail("send-email", {
+    to: <recipient-email-address>, // Recipient's email address.
+    subject: <email-subject>, // Email subject.
+    text: <email-body>, // Plain text content of the email.
+    from: "Your-Name <your-email-address>", // Sender's email address and name.
+  });
+}
+```
+
+## Tasks
+
+| Function Name   | Description                      |
+| --------------- | -------------------------------- |
+| `emails.send`   | Send an email                    |
+| `emails.create` | Create an email                  |
+| `emails.get`    | Get an email                     |
+| `batch.send`    | Send a batch of emails at once   |
+| `batch.create`  | Create a batch of emails at once |
+
+## Example
+
+In this example we use [Zod](/documentation/guides/zod), a TypeScript-first schema declaration and validation library.
+
+```ts
+import { Resend } from "@trigger.dev/resend";
+import { Job, eventTrigger } from "@trigger.dev/sdk";
+import { z } from "zod";
+
+...
+
+const resend = new Resend({
+  id: "resend",
+  apiKey: process.env.RESEND_API_KEY!,
+});
+
+client.defineJob({
+  id: "send-resend-email",
+  name: "Send Resend Email",
+  version: "0.1.0",
+  trigger: eventTrigger({
+    name: "send.email",
+    schema: z.object({
+      to: z.union([z.string(), z.array(z.string())]),
+      subject: z.string(),
+      text: z.string(),
+    }),
+  }),
+  integrations: {
+    resend,
+  },
+  run: async (payload, io, ctx) => {
+    await io.resend.emails.send("send-email", {
+      to: payload.to,
+      subject: payload.subject,
+      text: payload.text,
+      from: "Trigger.dev <hello@email.trigger.dev>",
+    });
+  },
+});
+```

docs/integrations/apis/resend.mdx

@@ -1,10 +1,23 @@
 ---
-title: Resend
+title: Resend overview & authentication
+sidebarTitle: Overview & authentication
 ---
 
-<Snippet file="integration-getting-started.mdx" />
+## Overview
 
-## Installation
+Resend is the email API for developers. With our Resend integration you can send email campaigns, transactional emails, and automated emails (drip campaigns) from your app.
+
+<Card
+  title="Jobs Showcase - Resend"
+  icon="rocket"
+  href="https://trigger.dev/showcase?tags=&apis=resend"
+>
+  Check out pre-built Resend jobs in our showcase.
+</Card>
+
+## Installing the Resend packages
+
+To get started with our Resend integration, you need to install the `@trigger.dev/resend` packages. You can do this using `npm`, `pnpm`, or `yarn`:
 
 <CodeGroup>
 
@@ -24,7 +37,7 @@ yarn add @trigger.dev/resend@latest
 
 ## Authentication
 
-Resend supports API Keys
+Resend supports API Keys. Official documentation [here](https://resend.com/docs/api-reference/api-keys/create-api-key).
 
 ```ts
 import { Resend } from "@trigger.dev/resend";
@@ -35,54 +48,10 @@ const resend = new Resend({
 });
 ```
 
-## Example
-
-In this example we use [Zod](/documentation/guides/zod), a TypeScript-first schema declaration and validation library.
-
-```ts
-import { Resend } from "@trigger.dev/resend";
-import { Job, eventTrigger } from "@trigger.dev/sdk";
-import { z } from "zod";
-
-...
-
-const resend = new Resend({
-  id: "resend",
-  apiKey: process.env.RESEND_API_KEY!,
-});
-
-client.defineJob({
-  id: "send-resend-email",
-  name: "Send Resend Email",
-  version: "0.1.0",
-  trigger: eventTrigger({
-    name: "send.email",
-    schema: z.object({
-      to: z.union([z.string(), z.array(z.string())]),
-      subject: z.string(),
-      text: z.string(),
-    }),
-  }),
-  integrations: {
-    resend,
-  },
-  run: async (payload, io, ctx) => {
-    await io.resend.emails.send("send-email", {
-      to: payload.to,
-      subject: payload.subject,
-      text: payload.text,
-      from: "Trigger.dev <hello@email.trigger.dev>",
-    });
-  },
-});
-```
-
 ## Tasks
 
-| Function Name   | Description                      |
-| --------------- | -------------------------------- |
-| `emails.send`   | Send an email                    |
-| `emails.create` | Create an email                  |
-| `emails.get`    | Get an email                     |
-| `batch.send`    | Send a batch of emails at once   |
-| `batch.create`  | Create a batch of emails at once |
+Once you have set up a Resend client, you can use it to create tasks.
+
+<Card title="Tasks" icon="sparkles" href="/integrations/apis/resend-tasks">
+  Send emails with Resend.
+</Card>

docs/integrations/apis/sendgrid-tasks.mdx

@@ -0,0 +1,89 @@
+---
+title: SendGrid tasks
+sidebarTitle: Tasks
+---
+
+Tasks are executed after the job is triggered and are the main building blocks of a job. You can string together as many tasks as you want.
+
+---
+
+## All tasks
+
+### `sendEmail`
+
+Send an email with a body. [Official SendGrid Docs](https://docs.sendgrid.com/for-developers/sending-email)
+
+```ts example.ts
+  // Using the 'io.sendgrid.sendEmail' method to send an email using SendGrid.
+  await io.sendgrid.sendEmail({
+    to: <recipient-email-address>, // Recipient's email address.
+    from: "Your-Name <your-email-address>", // Sender's email address and name.
+    subject: <email-subject>, // Email subject.
+    text: <email-body>, // Plain text content of the email.
+  });
+}
+
+```
+
+## Example usage
+
+In this example, we will send weekly summary emails to users who have `summariesEnabled = true`, at 4pm every Friday, and then posts the total numbers to Slack.
+
+```ts example.ts
+import { TriggerClient, cronTrigger } from "@trigger.dev/sdk";
+import { SendGrid } from "@trigger.dev/sendgrid";
+import { Slack } from "@trigger.dev/slack";
+import { weeklySummaryDb } from "./mocks/db";
+import { weeklySummaryEmail } from "./mocks/emails";
+
+// Creating instances of SendGrid and Slack clients.
+const sendgrid = new SendGrid({
+  id: "sendgrid",
+  apiKey: process.env.SENDGRID_API_KEY!,
+});
+
+const slack = new Slack({ id: "slack" });
+
+// Defining a job that sends a weekly summary email to users and posts total numbers to Slack.
+client.defineJob({
+  id: "weekly-user-activity-summary", // Unique identifier for the job.
+  name: "Weekly user activity summary", // A specific name for the job.
+  version: "1.0.0", // Version number for the job.
+  integrations: { sendgrid, slack }, // Integrating SendGrid and Slack clients into this job.
+  trigger: cronTrigger({
+    // Setting a cron schedule to run the job every Friday at 4 pm.
+    cron: "0 16 * * 5",
+  }),
+  run: async (payload, io, ctx) => {
+    // Inside the 'run' function, several operations are performed to send weekly summaries and post to Slack.
+
+    // Retrieving a list of users from the weeklySummaryDb.
+    const users = await weeklySummaryDb.getUsers();
+
+    let sentCount = 0;
+    let notSentCount = 0;
+
+    // Iterating through the list of users.
+    for (const user of users) {
+      if (user.summariesEnabled) {
+        // Sending a weekly summary email to users with summaries enabled.
+        await io.sendgrid.sendEmail(`Weekly summary for ${user.id}`, {
+          to: user.email,
+          from: "hello@acme.inc", // The sender's email.
+          subject: "Your weekly summary",
+          html: weeklySummaryEmail(user), // HTML content for the email.
+        });
+        sentCount++;
+      } else {
+        notSentCount++;
+      }
+    }
+
+    // Posting a message to Slack with a summary of the sent and unsent emails.
+    await io.slack.postMessage("Notify team", {
+      text: `Weekly summary sent to ${sentCount} users and not sent to ${notSentCount} users`,
+      channel: "YOUR_CHANNEL_ID", // Specify the Slack channel ID for posting the message.
+    });
+  },
+});
+```

docs/integrations/apis/sendgrid.mdx

@@ -1,10 +1,23 @@
 ---
-title: SendGrid
+title: SendGrid overview & authentication
+sidebarTitle: Overview & authentication
 ---
 
-<Snippet file="integration-getting-started.mdx" />
+## Overview
 
-## Installation
+SendGrid is a cloud-based SMTP provider that allows you to send email without having to maintain email servers. With our SendGrid integration you can send email campaigns, transactional emails, and automated emails (drip campaigns) from your app.
+
+<Card
+  title="Jobs Showcase - SendGrid"
+  icon="rocket"
+  href="https://trigger.dev/showcase?tags=&apis=sendgrid"
+>
+  Check out pre-built SendGrid jobs in our showcase.
+</Card>
+
+## Installing the SendGrid packages
+
+To get started with our SendGrid integration, you need to install the `@trigger.dev/sendgrid` packages. You can do this using `npm`, `pnpm`, or `yarn`:
 
 <CodeGroup>
 
@@ -24,7 +37,7 @@ yarn add @trigger.dev/sendgrid@latest
 
 ## Authentication
 
-SendGrid integration supports API Keys. To authenticate, you'll need to create an instance of the SendGrid class and provide your API key.
+SendGrid supports API Keys. Official documentation [here](https://docs.sendgrid.com/ui/account-and-settings/api-keys).
 
 ```ts
 import { SendGrid } from "@trigger.dev/sendgrid";
@@ -35,50 +48,12 @@ const sendgrid = new SendGrid({
 });
 ```
 
-## Example
-
-In this example we use [Zod](/documentation/guides/zod), a TypeScript-first schema declaration and validation library.
-
-```ts
-import { SendGrid } from "@trigger.dev/sendgrid";
-import { Job, eventTrigger } from "@trigger.dev/sdk";
-import { z } from "zod";
-
-// Create an instance of SendGrid
-const sendgrid = new SendGrid({
-  id: "sendgrid",
-  apiKey: process.env.SENDGRID_API_KEY!,
-});
-
-// Define a Trigger.dev job
-client.defineJob({
-  id: "send-sendgrid-email",
-  name: "Send SendGrid Email",
-  version: "0.1.0",
-  trigger: eventTrigger({
-    name: "send.email",
-    schema: z.object({
-      to: z.string(),
-      subject: z.string(),
-      text: z.string(),
-    }),
-  }),
-  integrations: {
-    sendgrid,
-  },
-  run: async (payload, io, ctx) => {
-    await io.sendgrid.sendEmail({
-      to: payload.to,
-      from: "Trigger.dev <hello@email.trigger.dev>",
-      subject: payload.subject,
-      text: payload.text,
-    });
-  },
-});
-```
-
 ## Tasks
 
-| Function Name | Description   |
-| ------------- | ------------- |
-| `sendEmail`   | Send an email |
+Once you have set up a SendGrid client, you can use it to create tasks.
+
+{" "}
+
+<Card title="Tasks" icon="sparkles" href="/integrations/apis/sendgrid-tasks">
+  Send emails with SendGrid.
+</Card>