Add support + docs for custom esbuild plugins (#1270)

* Add support + docs for custom esbuild plugins

* Fixed the missing subpath export

* Add support for specifying the prisma client generator

* Add `—env-file` support to dev and deploy

.changeset/rotten-eggs-occur.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/build": patch
+"@trigger.dev/core": patch
+---
+
+Added support for custom esbuild plugins

docs/guides/new-build-system-preview.mdx

@@ -39,6 +39,8 @@ If you've added the `trigger.dev` CLI to your `devDependencies`, then you should
 
 Once you do that make sure you re-install your dependencies using `npm i` or the equivalent with your preferred package manager.
 
+<Note>If you deploy using GitHub actions, make sure you update the version there too.</Note>
+
 ## Update your `trigger.config.ts`
 
 The new build system does not effect your trigger task files at all, so those can remain unchanged. However, you may need to make changes to your `trigger.config.ts` file.
@@ -229,6 +231,49 @@ export default defineConfig({
 });
 ```
 
+If you have multiple `generator` statements defined in your schema file, you can pass in the `clientGenerator` option to specify the `prisma-client-js` generator, which will prevent other generators from being generated:
+
+<CodeGroup>
+
+```prisma schema.prisma
+datasource db {
+  provider  = "postgresql"
+  url       = env("DATABASE_URL")
+  directUrl = env("DATABASE_URL_UNPOOLED")
+}
+
+// We only want to generate the prisma-client-js generator
+generator client {
+  provider        = "prisma-client-js"
+}
+
+generator kysely {
+  provider     = "prisma-kysely"
+  output       = "../../src/kysely"
+  enumFileName = "enums.ts"
+  fileName     = "types.ts"
+}
+```
+
+```ts trigger.config.ts
+import { defineConfig } from "@trigger.dev/sdk/v3";
+import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
+
+export default defineConfig({
+  project: "<project ref>",
+  build: {
+    extensions: [
+      prismaExtension({
+        schema: "prisma/schema.prisma",
+        clientGenerator: "client",
+      }),
+    ],
+  },
+});
+```
+
+</CodeGroup>
+
 ### audioWaveform
 
 Previously, we installed [Audio Waveform](https://github.com/bbc/audiowaveform) in the build image. That's been moved to a build extension:
@@ -245,6 +290,115 @@ export default defineConfig({
 });
 ```
 
+### esbuild plugins
+
+You can now add esbuild plugins to customize the build process using the `esbuildPlugin` build extension. The example below shows how to automatically upload sourcemaps to Sentry using their esbuild plugin:
+
+```ts
+import { defineConfig } from "@trigger.dev/sdk/v3";
+import { esbuildPlugin } from "@trigger.dev/build/extensions";
+import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
+
+export default defineConfig({
+  project: "<project ref>",
+  build: {
+    extensions: [
+      esbuildPlugin(
+        sentryEsbuildPlugin({
+          org: process.env.SENTRY_ORG,
+          project: process.env.SENTRY_PROJECT,
+          authToken: process.env.SENTRY_AUTH_TOKEN,
+        }),
+        // optional - only runs during the deploy command, and adds the plugin to the end of the list of plugins
+        { placement: "last", target: "deploy" }
+      ),
+    ],
+  },
+});
+```
+
+## Changes to the `trigger.dev` CLI
+
+### No more typechecking during deploy
+
+We no longer run typechecking during the deploy command. This was causing issues with some projects, and we found that it wasn't necessary to run typechecking during the deploy command. If you want to run typechecking before deploying to Trigger.dev, you can run the `tsc` command before running the `deploy` command.
+
+```sh
+tsc && npx trigger.dev@0.0.0-prerelease-20240823132052 deploy
+```
+
+Or if you are using GitHub actions, you can add an additional step to run the `tsc` command before deploying to Trigger.dev.
+
+```yaml
+- name: Install dependencies
+  run: npm install
+
+- name: Typecheck
+  run: npx tsc
+
+- name: 🚀 Deploy Trigger.dev
+  env:
+    TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }}
+  run: |
+    npx trigger.dev@0.0.0-prerelease-20240823132052 deploy
+```
+
+### deploy --dry-run
+
+You can now inspect the build output of your project without actually deploying it to Trigger.dev by using the `--dry-run` flag:
+
+```sh
+npx trigger.dev@0.0.0-prerelease-20240823132052 deploy --dry-run
+```
+
+This will save the build output and print the path to the build output directory. If you face any issues with deploying, please include the build output in your issue report.
+
+### --env-file
+
+You can now pass the path to your local `.env` file using the `--env-file` flag during `dev` and `deploy` commands:
+
+```sh
+npx trigger.dev@0.0.0-prerelease-20240823132052 dev --env-file ../../.env
+npx trigger.dev@0.0.0-prerelease-20240823132052 deploy --env-file ../../.env
+```
+
+The `.env` file works slightly differently in `dev` vs `deploy`:
+
+- In `dev`, the `.env` file is loaded into the CLI's `process.env` and also into the environment variables of the Trigger.dev environment.
+- In `deploy`, the `.env` file is loaded into the CLI's `process.env` but not into the environment variables of the Trigger.dev environment. If you want to sync the environment variables from the `.env` file to the Trigger.dev environment variables, you can use the `syncEnvVars` build extension.
+
+### dev debugging in VS Code
+
+Debugging your tasks code in `dev` is now supported via VS Code, without having to pass in any additional flags. Create a launch configuration in `.vscode/launch.json`:
+
+```json launch.json
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Trigger.dev: Dev",
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "runtimeExecutable": "npx",
+      "runtimeArgs": ["trigger.dev@0.0.0-prerelease-20240823132052", "dev"],
+      "skipFiles": ["<node_internals>/**"],
+      "sourceMaps": true
+    }
+  ]
+}
+```
+
+Then you can start debugging your tasks code by selecting the `Trigger.dev: Dev` configuration in the debug panel, and set breakpoints in your tasks code.
+
+### TRIGGER_ACCESS_TOKEN in dev
+
+You can now authenticate the `dev` command using the `TRIGGER_ACCESS_TOKEN` environment variable. Previously this was only supported in the `deploy` command.
+
+```sh
+TRIGGER_ACCESS_TOKEN=<your access token> npx trigger.dev@0.0.0-prerelease-20240823132052 dev
+```
+
 ## Known issues
 
 - Path aliases are not yet support in your `trigger.config.ts` file. To workaround this issue you'll need to rewrite path aliases to their relative paths. (See [this](https://github.com/unjs/jiti/issues/166) and [this](https://knip.dev/reference/known-issues#path-aliases-in-config-files)) for more info.

packages/build/package.json

@@ -23,6 +23,7 @@
     "exports": {
       "./package.json": "./package.json",
       ".": "./src/index.ts",
+      "./extensions": "./src/extensions/index.ts",
       "./extensions/core": "./src/extensions/core.ts",
       "./extensions/prisma": "./src/extensions/prisma.ts",
       "./extensions/audioWaveform": "./src/extensions/audioWaveform.ts",
@@ -68,6 +69,17 @@
         "default": "./dist/commonjs/index.js"
       }
     },
+    "./extensions": {
+      "import": {
+        "@triggerdotdev/source": "./src/extensions/index.ts",
+        "types": "./dist/esm/extensions/index.d.ts",
+        "default": "./dist/esm/extensions/index.js"
+      },
+      "require": {
+        "types": "./dist/commonjs/extensions/index.d.ts",
+        "default": "./dist/commonjs/extensions/index.js"
+      }
+    },
     "./extensions/core": {
       "import": {
         "@triggerdotdev/source": "./src/extensions/core.ts",

packages/build/src/extensions/core/additionalPackages.ts

@@ -41,14 +41,16 @@ export function additionalPackages(options: AdditionalPackagesOptions): BuildExt
               continue;
             }
 
-            console.log("Resolved module path", { modulePath });
+            context.logger.debug("[additionalPackages] Resolved module path", { modulePath });
 
             const packageJSON = await readPackageJSON(dirname(modulePath));
 
             if (packageJSON.version) {
               dependencies[name] = packageJSON.version;
             } else {
-              console.warn(`Could not resolve version for package ${name}, defaulting to latest`);
+              context.logger.warn(
+                `Could not resolve version for package ${name}, defaulting to latest`
+              );
 
               dependencies[name] = "latest";
             }

packages/build/src/extensions/index.ts

@@ -0,0 +1 @@
+export * from "@trigger.dev/core/v3/build";

packages/build/src/extensions/prisma.ts

@@ -9,6 +9,34 @@ export type PrismaExtensionOptions = {
   schema: string;
   migrate?: boolean;
   version?: string;
+  /**
+   * The client generator to use. Set this param to prevent all generators in the prisma schema from being generated.
+   *
+   * @example
+   *
+   * ### Prisma schema
+   *
+   * ```prisma
+   * generator client {
+   *  provider = "prisma-client-js"
+   * }
+   *
+   * generator typegraphql {
+   *  provider = "typegraphql-prisma"
+   *  output = "./generated/type-graphql"
+   * }
+   * ```
+   *
+   * ### PrismaExtension
+   *
+   * ```ts
+   * prismaExtension({
+   *  schema: "./prisma/schema.prisma",
+   *  clientGenerator: "client"
+   * });
+   * ```
+   */
+  clientGenerator?: string;
   directUrlEnvVarName?: string;
 };
 
@@ -86,6 +114,10 @@ export class PrismaExtension implements BuildExtension {
 
     let prismaDir: string | undefined;
 
+    const generatorFlag = this.options.clientGenerator
+      ? `--generator=${this.options.clientGenerator}`
+      : "";
+
     if (usingSchemaFolder) {
       const schemaDir = dirname(this._resolvedSchemaPath);
 
@@ -116,7 +148,9 @@ export class PrismaExtension implements BuildExtension {
       }
 
       commands.push(
-        `${binaryForRuntime(manifest.runtime)} node_modules/prisma/build/index.js generate` // Don't add the --schema flag or this will fail
+        `${binaryForRuntime(
+          manifest.runtime
+        )} node_modules/prisma/build/index.js generate ${generatorFlag}` // Don't add the --schema flag or this will fail
       );
     } else {
       prismaDir = dirname(this._resolvedSchemaPath);
@@ -135,7 +169,7 @@ export class PrismaExtension implements BuildExtension {
       commands.push(
         `${binaryForRuntime(
           manifest.runtime
-        )} node_modules/prisma/build/index.js generate --schema=./prisma/schema.prisma`
+        )} node_modules/prisma/build/index.js generate --schema=./prisma/schema.prisma ${generatorFlag}`
       );
     }
 

packages/build/src/extensions/typescript.ts

@@ -1,4 +1,4 @@
-import { BuildExtension, createExtensionForPlugin } from "@trigger.dev/core/v3/build";
+import { BuildExtension, esbuildPlugin } from "@trigger.dev/core/v3/build";
 import type { Plugin } from "esbuild";
 import { readFile } from "node:fs/promises";
 import { readTSConfig } from "pkg-types";
@@ -13,7 +13,7 @@ export type EmitDecoratorMetadataOptions = {
 };
 
 export function emitDecoratorMetadata(options: EmitDecoratorMetadataOptions = {}): BuildExtension {
-  return createExtensionForPlugin(plugin(options));
+  return esbuildPlugin(plugin(options));
 }
 
 function plugin(options: EmitDecoratorMetadataOptions = {}): Plugin {

packages/cli-v3/src/commands/deploy.ts

@@ -69,6 +69,7 @@ const DeployCommandOptions = CommonCommandOptions.extend({
   saveLogs: z.boolean().default(false),
   skipUpdateCheck: z.boolean().default(false),
   noCache: z.boolean().default(false),
+  envFile: z.string().optional(),
 });
 
 type DeployCommandOptions = z.infer<typeof DeployCommandOptions>;
@@ -100,6 +101,10 @@ export function configureDeployCommand(program: Command) {
         "--skip-sync-env-vars",
         "Skip syncing environment variables when using the syncEnvVars extension."
       )
+      .option(
+        "--env-file <env file>",
+        "Path to the .env file to load into the CLI process. Defaults to .env in the project directory."
+      )
   )
     .addOption(
       new CommandOption(
@@ -209,7 +214,7 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
   }
 
   const serverEnvVars = await projectClient.client.getEnvironmentVariables(resolvedConfig.project);
-  loadDotEnvVars(resolvedConfig.workingDir);
+  loadDotEnvVars(resolvedConfig.workingDir, options.envFile);
 
   const destination = getTmpDir(resolvedConfig.workingDir, "build", options.dryRun);
   const externalsExtension = createExternalsBuildExtension("deploy", resolvedConfig);

packages/cli-v3/src/commands/dev.ts

@@ -17,6 +17,7 @@ const DevCommandOptions = CommonCommandOptions.extend({
   config: z.string().optional(),
   projectRef: z.string().optional(),
   skipUpdateCheck: z.boolean().default(false),
+  envFile: z.string().optional(),
 });
 
 export type DevCommandOptions = z.infer<typeof DevCommandOptions>;
@@ -32,6 +33,10 @@ export function configureDevCommand(program: Command) {
         "-p, --project-ref <project ref>",
         "The project ref. Required if there is no config file."
       )
+      .option(
+        "--env-file <env file>",
+        "Path to the .env file to use for the dev session. Defaults to .env in the project directory."
+      )
       .option("--debug-otel", "Enable OpenTelemetry debugging")
       .option("--skip-update-check", "Skip checking for @trigger.dev package updates")
   ).action(async (_, options) => {

packages/cli-v3/src/dev/workerRuntime.ts

@@ -229,7 +229,7 @@ class DevWorkerRuntime implements WorkerRuntime {
     );
 
     const processEnv = gatherProcessEnv();
-    const dotEnvVars = resolveDotEnvVars();
+    const dotEnvVars = resolveDotEnvVars(undefined, this.options.args.envFile);
     const OTEL_IMPORT_HOOK_INCLUDES = getInstrumentedPackageNames(this.options.config).join(",");
 
     const stripEmptyValues = (obj: Record<string, string | undefined>) => {

packages/cli-v3/src/utilities/dotEnv.ts

@@ -4,12 +4,16 @@ import { env } from "std-env";
 
 const ENVVAR_FILES = [".env", ".env.development", ".env.local", ".env.development.local"];
 
-export function resolveDotEnvVars(cwd?: string) {
+export function resolveDotEnvVars(cwd?: string, envFile?: string) {
   const result: { [key: string]: string } = {};
 
+  const envFilePath = envFile
+    ? resolve(cwd ?? process.cwd(), envFile)
+    : ENVVAR_FILES.map((p) => resolve(cwd ?? process.cwd(), p));
+
   dotenv.config({
     processEnv: result,
-    path: ENVVAR_FILES.map((p) => resolve(cwd ?? process.cwd(), p)),
+    path: envFilePath,
   });
 
   env.TRIGGER_API_URL && (result.TRIGGER_API_URL = env.TRIGGER_API_URL);
@@ -21,8 +25,12 @@ export function resolveDotEnvVars(cwd?: string) {
   return result;
 }
 
-export function loadDotEnvVars(cwd?: string) {
+export function loadDotEnvVars(cwd?: string, envFile?: string) {
+  const envFilePath = envFile
+    ? resolve(cwd ?? process.cwd(), envFile)
+    : ENVVAR_FILES.map((p) => resolve(cwd ?? process.cwd(), p));
+
   dotenv.config({
-    path: ENVVAR_FILES.map((p) => resolve(cwd ?? process.cwd(), p)),
+    path: envFilePath,
   });
 }

packages/core/src/v3/build/extensions.ts

@@ -2,10 +2,7 @@ import { BuildManifest, BuildTarget } from "../schemas/build.js";
 import type { Plugin } from "esbuild";
 import { ResolvedConfig } from "./resolvedConfig.js";
 
-export function createExtensionForPlugin(
-  plugin: Plugin,
-  options: RegisterPluginOptions = {}
-): BuildExtension {
+export function esbuildPlugin(plugin: Plugin, options: RegisterPluginOptions = {}): BuildExtension {
   return {
     name: plugin.name,
     onBuildStart(context) {

references/v3-catalog/package.json

@@ -13,16 +13,18 @@
     "build:client": "tsup-node ./src/clientUsage.ts --format esm,cjs",
     "client": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/clientUsage.ts",
     "triggerWithLargePayload": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/triggerWithLargePayload.ts",
-    "postinstall": "prisma generate --no-hints"
+    "generate": "prisma generate",
+    "postinstall": "pnpm run generate"
   },
   "dependencies": {
-    "@prisma/client": "5.18.0",
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "@ffprobe-installer/ffprobe": "^2.1.2",
     "@infisical/sdk": "^2.1.9",
     "@opentelemetry/api": "1.4.1",
+    "@prisma/client": "5.18.0",
     "@react-email/components": "^0.0.17",
     "@react-email/render": "^0.0.7",
+    "@sentry/esbuild-plugin": "^2.22.2",
     "@sindresorhus/slugify": "^2.2.1",
     "@t3-oss/env-core": "^0.11.0",
     "@t3-oss/env-nextjs": "^0.10.1",
@@ -31,6 +33,7 @@
     "dotenv": "^16.4.5",
     "email-reply-parser": "^1.8.0",
     "execa": "^8.0.1",
+    "kysely": "^0.27.4",
     "msw": "^2.2.1",
     "openai": "^4.47.0",
     "pg": "^8.11.5",
@@ -60,15 +63,16 @@
     "@opentelemetry/sdk-trace-base": "^1.22.0",
     "@opentelemetry/sdk-trace-node": "^1.22.0",
     "@opentelemetry/semantic-conventions": "^1.22.0",
+    "@trigger.dev/build": "workspace:*",
     "@types/email-reply-parser": "^1.4.2",
     "@types/node": "20.4.2",
     "@types/react": "^18.3.1",
     "esbuild": "^0.19.11",
+    "prisma": "5.18.0",
+    "prisma-kysely": "^1.8.0",
     "trigger.dev": "workspace:*",
-    "@trigger.dev/build": "workspace:*",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.5.4",
-    "prisma": "5.18.0"
+    "typescript": "^5.5.4"
   }
 }

references/v3-catalog/prisma/schema/schema.prisma

@@ -14,3 +14,10 @@ datasource db {
   url       = env("DATABASE_URL")
   directUrl = env("DATABASE_URL_UNPOOLED")
 }
+
+generator kysely {
+  provider     = "prisma-kysely"
+  output       = "../../src/kysely"
+  enumFileName = "enums.ts"
+  fileName     = "types.ts"
+}

references/v3-catalog/src/kysely/types.ts

@@ -0,0 +1,20 @@
+import type { ColumnType } from "kysely";
+export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
+  ? ColumnType<S, I | undefined, U>
+  : ColumnType<T, T | undefined, T>;
+export type Timestamp = ColumnType<Date, Date | string, Date | string>;
+
+export type Post = {
+  id: Generated<number>;
+  title: string;
+  content: string;
+  authorId: number;
+};
+export type User = {
+  id: Generated<number>;
+  name: string;
+};
+export type DB = {
+  Post: Post;
+  User: User;
+};

references/v3-catalog/trigger.config.ts

@@ -1,9 +1,11 @@
 import { InfisicalClient } from "@infisical/sdk";
 import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
+import { esbuildPlugin } from "@trigger.dev/build/extensions";
 import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform";
 import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
 import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript";
 import { defineConfig, ResolveEnvironmentVariablesFunction } from "@trigger.dev/sdk/v3";
+import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
 
 export { handleError } from "./src/handleError.js";
 
@@ -66,7 +68,16 @@ export default defineConfig({
         schema: "prisma/schema/schema.prisma",
         migrate: true,
         directUrlEnvVarName: "DATABASE_URL_UNPOOLED",
+        clientGenerator: "client",
       }),
+      esbuildPlugin(
+        sentryEsbuildPlugin({
+          org: "triggerdev",
+          project: "taskhero-examples-basic",
+          authToken: process.env.SENTRY_AUTH_TOKEN,
+        }),
+        { placement: "last", target: "deploy" }
+      ),
     ],
     external: ["@ffmpeg-installer/ffmpeg", "re2"],
   },