Add experimental externals detection (#2083)

* add auto detect externals

* stop spinner on esbuild errors

* test problematic packages

* fix braces types

* improve detection 10x

* ignore sentry stub

* improve main package json detection

* rename to experimental_autoDetectExternal

.changeset/grumpy-wasps-fold.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Add `experimental_autoDetectExternal` trigger config option

packages/cli-v3/package.json

@@ -96,6 +96,7 @@
     "@trigger.dev/build": "workspace:4.0.0-v4-beta.16",
     "@trigger.dev/core": "workspace:4.0.0-v4-beta.16",
     "ansi-escapes": "^7.0.0",
+    "braces": "^3.0.3",
     "c12": "^1.11.1",
     "chalk": "^5.2.0",
     "chokidar": "^3.6.0",

packages/cli-v3/src/build/braces.d.ts

@@ -0,0 +1,126 @@
+/*
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
+ */
+
+type Transform = (str: string) => string;
+
+interface Options {
+  /**
+   * Limit the length of the input string. Useful when the input string is generated or your application allows
+   * users to pass a string, et cetera.
+   *
+   * @default 65536
+   * @example
+   * console.log(braces('a/{b,c}/d', { maxLength: 3 }));
+   * //=> throws an error
+   */
+  maxLength?: number | undefined;
+  /**
+   * Generate an "expanded" brace pattern (alternatively you can use the `braces.expand()` method).
+   *
+   * @default undefined
+   * @example
+   * console.log(braces('a/{b,c}/d', { expand: true }));
+   * //=> [ 'a/b/d', 'a/c/d' ]
+   */
+  expand?: boolean | undefined;
+  /**
+   * Remove duplicates from the returned array.
+   *
+   * @default undefined
+   */
+  nodupes?: boolean | undefined;
+  /**
+   * To prevent malicious patterns from being passed by users, an error is thrown when `braces.expand()`
+   * is used or `options.expand` is true and the generated range will exceed the `rangeLimit`.
+   *
+   * You can customize `options.rangeLimit` or set it to `Infinity` to disable this altogether.
+   *
+   * @default 1000
+   * @example
+   * // pattern exceeds the "rangeLimit", so it's optimized automatically
+   * console.log(braces.expand('{1..1000}'));
+   * //=> ['([1-9]|[1-9][0-9]{1,2}|1000)']
+   *
+   * // pattern does not exceed "rangeLimit", so it's NOT optimized
+   * console.log(braces.expand('{1..100}'));
+   * //=> ['1', '2', '3', '4', '5', …, '100']
+   */
+  rangeLimit?: number | undefined;
+  /**
+   * Customize range expansion.
+   *
+   * @default undefined
+   * @example
+   * const range = braces.expand('x{a..e}y', {
+   *   transform: (str) => `foo/${str}`
+   * });
+   *
+   * console.log(range);
+   * //=> [ 'xfooay', 'xfooby', 'xfoocy', 'xfoody', 'xfooey' ]
+   */
+  transform?: Transform | undefined;
+  /**
+   * In regular expressions, quanitifiers can be used to specify how many times a token can be repeated.
+   * For example, `a{1,3}` will match the letter `a` one to three times.
+   *
+   * Unfortunately, regex quantifiers happen to share the same syntax as [Bash lists](#lists)
+   *
+   * The `quantifiers` option tells braces to detect when [regex quantifiers](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp#quantifiers)
+   * are defined in the given pattern, and not to try to expand them as lists.
+   *
+   * @default undefined
+   * @example
+   * const braces = require('braces');
+   * console.log(braces('a/b{1,3}/{x,y,z}'));
+   * //=> [ 'a/b(1|3)/(x|y|z)' ]
+   * console.log(braces('a/b{1,3}/{x,y,z}', {quantifiers: true}));
+   * //=> [ 'a/b{1,3}/(x|y|z)' ]
+   * console.log(braces('a/b{1,3}/{x,y,z}', {quantifiers: true, expand: true}));
+   * //=> [ 'a/b{1,3}/x', 'a/b{1,3}/y', 'a/b{1,3}/z' ]
+   */
+  quantifiers?: boolean | undefined;
+  /**
+   * Do not strip backslashes that were used for escaping from the result.
+   *
+   * @default undefined
+   */
+  keepEscaping?: boolean | undefined;
+  /**
+   * Do not strip quotes from the result.
+   *
+   * @default undefined
+   */
+  keepQuotes?: boolean | undefined;
+}
+
+// Ambient type override for braces to allow string or string[] as pattern
+declare module "braces" {
+  function braces(pattern: string | string[], options?: Options): string[];
+
+  namespace braces {
+    function expand(pattern: string | string[], options?: Omit<Options, "expand">): string[];
+  }
+
+  export default braces;
+}

packages/cli-v3/src/build/externals.ts

@@ -1,6 +1,6 @@
 import * as esbuild from "esbuild";
 import { makeRe } from "minimatch";
-import { mkdir, symlink } from "node:fs/promises";
+import { access, mkdir, symlink } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { readPackageJSON, resolvePackageJSON } from "pkg-types";
 import nodeResolve from "resolve";
@@ -14,6 +14,10 @@ import {
 import { logger } from "../utilities/logger.js";
 import { CliApiClient } from "../apiClient.js";
 import { resolvePathSync as esmResolveSync } from "mlly";
+import braces from "braces";
+import { builtinModules } from "node:module";
+import { tryCatch } from "@trigger.dev/core/v3";
+import { resolveModule } from "./resolveModule.js";
 
 /**
  * externals in dev might not be resolvable from the worker directory
@@ -140,6 +144,11 @@ function createExternalsCollector(
 
   const maybeExternals = discoverMaybeExternals(target, resolvedConfig, forcedExternal);
 
+  // Cache: resolvedPath (dir) -> packageJsonPath (null = failed to resolve)
+  const packageJsonCache = new Map<string, string | null>();
+  // Cache: packageRoot (dir) -> boolean (true = mark as external)
+  const isExternalCache = new Map<string, boolean>();
+
   return {
     externals,
     plugin: {
@@ -147,10 +156,17 @@ function createExternalsCollector(
       setup: (build) => {
         build.onStart(async () => {
           externals.splice(0);
+          isExternalCache.clear();
         });
 
         build.onEnd(async () => {
-          logger.debug("[externals][onEnd] Collected externals", { externals });
+          logger.debug("[externals][onEnd] Collected externals", {
+            externals,
+            maybeExternals,
+            autoDetectExternal: !!resolvedConfig.build?.experimental_autoDetectExternal,
+            packageJsonCache: packageJsonCache.size,
+            isExternalCache: isExternalCache.size,
+          });
         });
 
         maybeExternals.forEach((external) => {
@@ -248,6 +264,146 @@ function createExternalsCollector(
             }
           });
         });
+
+        if (resolvedConfig.build?.experimental_autoDetectExternal) {
+          build.onResolve(
+            { filter: /.*/, namespace: "file" },
+            async (args: esbuild.OnResolveArgs): Promise<esbuild.OnResolveResult | undefined> => {
+              if (!isBareModuleImport(args.path)) {
+                // Not an npm package
+                return;
+              }
+
+              if (isBuiltinModule(args.path)) {
+                // Builtin module
+                return;
+              }
+
+              if (args.path === "_sentry-debug-id-injection-stub") {
+                // Ignore sentry stub
+                return;
+              }
+
+              // Try to resolve the actual file path
+              const [resolveError, resolvedPath] = await tryCatch(
+                resolveModule(args.path, args.resolveDir)
+              );
+
+              if (resolveError) {
+                logger.debug("[externals][auto] Resolve module error", {
+                  path: args.path,
+                  resolveError,
+                });
+                return;
+              }
+
+              // Find nearest package.json
+              const packageJsonPath = await findNearestPackageJson(resolvedPath, packageJsonCache);
+
+              if (!packageJsonPath) {
+                logger.debug("[externals][auto] Failed to resolve package.json path", {
+                  path: args.path,
+                  resolvedPath,
+                });
+                return;
+              }
+
+              const packageRoot = dirname(packageJsonPath);
+
+              // Check cache first
+              if (isExternalCache.has(packageRoot)) {
+                const isExternal = isExternalCache.get(packageRoot);
+
+                if (isExternal) {
+                  return { path: args.path, external: true };
+                }
+
+                return;
+              }
+
+              const [readError, packageJson] = await tryCatch(readPackageJSON(packageRoot));
+
+              if (readError) {
+                logger.debug("[externals][auto] Unable to read package.json", {
+                  error: readError,
+                  packageRoot,
+                });
+
+                isExternalCache.set(packageRoot, false);
+                return;
+              }
+
+              const packageName = packageJson.name;
+              const packageVersion = packageJson.version;
+
+              if (!packageName || !packageVersion) {
+                logger.debug("[externals][auto] No package name or version found in package.json", {
+                  packageRoot,
+                  packageJson,
+                });
+
+                return;
+              }
+
+              const markExternal = (reason: string): esbuild.OnResolveResult => {
+                const detectedPackage = {
+                  name: packageName,
+                  path: packageRoot,
+                  version: packageVersion,
+                } satisfies CollectedExternal;
+
+                logger.debug(`[externals][auto] Marking as external - ${reason}`, {
+                  detectedPackage,
+                });
+
+                externals.push(detectedPackage);
+
+                // Cache the result
+                isExternalCache.set(packageRoot, true);
+
+                return { path: args.path, external: true };
+              };
+
+              // If the path ends with .wasm or .node, we should mark it as external
+              if (resolvedPath.endsWith(".wasm") || resolvedPath.endsWith(".node")) {
+                return markExternal("path ends with .wasm or .node");
+              }
+
+              // Check files, main, module fields for native files
+              const files = Array.isArray(packageJson.files) ? packageJson.files : [];
+              const fields = [packageJson.main, packageJson.module, packageJson.browser].filter(
+                (f): f is string => typeof f === "string"
+              );
+              const allFiles = files.concat(fields);
+
+              // We need to expand any braces in the files array, e.g. ["{js,ts}"] -> ["js", "ts"]
+              const allFilesExpanded = braces(allFiles, { expand: true });
+
+              // Use a regexp to match native-related extensions
+              const nativeExtRegexp = /\.(wasm|node|gyp|c|cc|cpp|cxx|h|hpp|hxx)$/;
+              const hasNativeFile = allFilesExpanded.some((file) => nativeExtRegexp.test(file));
+
+              if (hasNativeFile) {
+                return markExternal("has native file");
+              }
+
+              // Check if binding.gyp exists (native addon)
+              const bindingGypPath = join(packageRoot, "binding.gyp");
+
+              // If access succeeds, binding.gyp exists
+              const [accessError] = await tryCatch(access(bindingGypPath));
+
+              if (!accessError) {
+                return markExternal("binding.gyp exists");
+              }
+
+              // Cache the negative result
+              isExternalCache.set(packageRoot, false);
+
+              return undefined;
+            }
+          );
+        }
       },
     },
   };
@@ -410,3 +566,87 @@ function resolveSync(id: string, resolveDir: string) {
     return esmResolveSync(id, { url: resolveDir });
   }
 }
+
+function isBareModuleImport(path: string): boolean {
+  const excludes = [".", "/", "~", "file:", "data:"];
+  return !excludes.some((exclude) => path.startsWith(exclude));
+}
+
+function isBuiltinModule(path: string): boolean {
+  return builtinModules.includes(path.replace("node:", ""));
+}
+
+async function isMainPackageJson(filePath: string): Promise<boolean> {
+  try {
+    const packageJson = await readPackageJSON(filePath);
+
+    // Allowlist of non-informative fields that can appear with 'type: module | commonjs' in marker package.json files
+    const markerFields = new Set([
+      "type",
+      "sideEffects",
+      "browser",
+      "main",
+      "module",
+      "react-native",
+      "name",
+    ]);
+
+    if (!packageJson.type) {
+      return true;
+    }
+
+    const keys = Object.keys(packageJson);
+    if (keys.every((k) => markerFields.has(k))) {
+      return false; // type marker
+    }
+
+    return true;
+  } catch (error) {
+    if (!(error instanceof Error)) {
+      logger.debug("[externals][containsEsmTypeMarkers] Unknown error", {
+        error,
+      });
+
+      return false;
+    }
+
+    if ("code" in error && error.code !== "ENOENT") {
+      logger.debug("[externals][containsEsmTypeMarkers] Error", {
+        error: error.message,
+      });
+    }
+
+    return false;
+  }
+}
+
+async function findNearestPackageJson(
+  basePath: string,
+  cache: Map<string, string | null>
+): Promise<string | null> {
+  const baseDir = dirname(basePath);
+
+  if (cache.has(baseDir)) {
+    const resolvedPath = cache.get(baseDir);
+
+    if (!resolvedPath) {
+      return null;
+    }
+
+    return resolvedPath;
+  }
+
+  const [error, packageJsonPath] = await tryCatch(
+    resolvePackageJSON(dirname(basePath), {
+      test: isMainPackageJson,
+    })
+  );
+
+  if (error) {
+    cache.set(baseDir, null);
+    return null;
+  }
+
+  cache.set(baseDir, packageJsonPath);
+  return packageJsonPath;
+}

packages/cli-v3/src/commands/deploy.ts

@@ -1,5 +1,5 @@
 import { intro, log, outro } from "@clack/prompts";
-import { prepareDeploymentError } from "@trigger.dev/core/v3";
+import { prepareDeploymentError, tryCatch } from "@trigger.dev/core/v3";
 import { InitializeDeploymentResponseBody } from "@trigger.dev/core/v3/schemas";
 import { Command, Option as CommandOption } from "commander";
 import { resolve } from "node:path";
@@ -238,25 +238,32 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
 
   const { features } = resolvedConfig;
 
-  const buildManifest = await buildWorker({
-    target: "deploy",
-    environment: options.env,
-    destination: destination.path,
-    resolvedConfig,
-    rewritePaths: true,
-    envVars: serverEnvVars.success ? serverEnvVars.data.variables : {},
-    forcedExternals,
-    listener: {
-      onBundleStart() {
-        $buildSpinner.start("Building trigger code");
-      },
-      onBundleComplete(result) {
-        $buildSpinner.stop("Successfully built code");
+  const [error, buildManifest] = await tryCatch(
+    buildWorker({
+      target: "deploy",
+      environment: options.env,
+      destination: destination.path,
+      resolvedConfig,
+      rewritePaths: true,
+      envVars: serverEnvVars.success ? serverEnvVars.data.variables : {},
+      forcedExternals,
+      listener: {
+        onBundleStart() {
+          $buildSpinner.start("Building trigger code");
+        },
+        onBundleComplete(result) {
+          $buildSpinner.stop("Successfully built code");
 
-        logger.debug("Bundle result", result);
+          logger.debug("Bundle result", result);
+        },
       },
-    },
-  });
+    })
+  );
+
+  if (error) {
+    $buildSpinner.stop("Failed to build code");
+    throw error;
+  }
 
   logger.debug("Successfully built project to", destination.path);
 

packages/core/src/v3/config.ts

@@ -170,6 +170,19 @@ export type TriggerConfig = {
      */
     external?: string[];
 
+    /**
+     * **WARNING: This is an experimental feature and might be removed in a future version.**
+     *
+     * Automatically detect dependencies that shouldn't be bundled and mark them as external. For example, native modules.
+     *
+     * Turning this on will not affect dependencies that were manually added to the `external` array.
+     *
+     * @default false
+     *
+     * @deprecated (experimental)
+     */
+    experimental_autoDetectExternal?: boolean;
+
     jsx?: {
       /**
        * @default "React.createElement"

references/v3-catalog/package.json

@@ -16,8 +16,9 @@
     "generate:prisma": "prisma generate --sql"
   },
   "dependencies": {
+    "@1password/sdk": "^0.3.0",
     "@effect/schema": "^0.75.5",
-    "@infisical/sdk": "^2.1.9",
+    "@infisical/sdk": "^2.3.5",
     "@opentelemetry/api": "1.4.1",
     "@prisma/client": "5.19.0",
     "@react-email/components": "0.0.24",
@@ -32,6 +33,8 @@
     "@typeschema/typebox": "^0.14.0",
     "ai": "^3.3.24",
     "arktype": "2.0.0-rc.17",
+    "bcrypt": "^6.0.0",
+    "canvas": "^3.1.0",
     "dotenv": "^16.4.5",
     "email-reply-parser": "^1.8.0",
     "execa": "^8.0.1",
@@ -39,6 +42,7 @@
     "header-generator": "^2.1.55",
     "kysely": "^0.27.4",
     "msw": "^2.2.1",
+    "mupdf": "^1.3.6",
     "openai": "^4.47.0",
     "pg": "^8.11.5",
     "playwright": "^1.50.1",
@@ -48,6 +52,8 @@
     "reflect-metadata": "^0.1.13",
     "runtypes": "^6.7.0",
     "server-only": "^0.0.1",
+    "sharp": "^0.34.2",
+    "sqlite3": "^5.1.7",
     "stripe": "^12.14.0",
     "superstruct": "^2.0.2",
     "typeorm": "^0.3.20",
@@ -55,6 +61,7 @@
     "wrangler": "3.70.0",
     "yt-dlp-wrap": "^2.3.12",
     "yup": "^1.4.0",
+    "zip-node-addon": "^0.0.11",
     "zod": "3.23.8"
   },
   "devDependencies": {
@@ -75,6 +82,7 @@
     "@opentelemetry/semantic-conventions": "^1.22.0",
     "@trigger.dev/build": "workspace:*",
     "@trigger.dev/python": "workspace:*",
+    "@types/bcrypt": "^5.0.2",
     "@types/email-reply-parser": "^1.4.2",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/react": "^18.3.1",

references/v3-catalog/src/trigger/binaries.ts

@@ -4,6 +4,87 @@ import * as path from "node:path";
 import { Readable } from "node:stream";
 import type { ReadableStream } from "node:stream/web";
 
+import bcrypt from "bcrypt";
+
+const saltRounds = 10;
+const myPlaintextPassword = "s0//P4$$w0rD";
+const someOtherPlaintextPassword = "not_bacon";
+
+bcrypt.genSalt(saltRounds, function (err, salt) {
+  bcrypt.hash(myPlaintextPassword, salt, function (err, hash) {
+    // Store hash in your password DB.
+  });
+});
+
+import { InfisicalClient } from "@infisical/sdk";
+
+const infisicalClient = new InfisicalClient({
+  siteUrl: "https://example.com",
+});
+
+import * as mupdf from "mupdf";
+
+// Helper function to load document from URL
+async function loadDocumentFromUrl(url: string): Promise<mupdf.Document> {
+  try {
+    const response = await fetch(url);
+    const buffer = await response.arrayBuffer();
+    return mupdf.Document.openDocument(buffer, "application/pdf");
+  } catch (error) {
+    throw new Error(`Failed to load document from URL: ${url}`);
+  }
+}
+
+import zip from "zip-node-addon";
+
+function unzip(inputPath: string, outputPath: string) {
+  zip.unzipFile(inputPath, outputPath);
+}
+
+import { createClient } from "@1password/sdk";
+
+// Creates an authenticated client.
+const client = await createClient({
+  auth: process.env.OP_SERVICE_ACCOUNT_TOKEN ?? "",
+  // Set the following to your own integration name and version.
+  integrationName: "My 1Password Integration",
+  integrationVersion: "v1.0.0",
+});
+
+// Fetches a secret.
+// const secret = await client.secrets.resolve("op://vault/item/field");
+
+import sharp from "sharp";
+import sqlite3 from "sqlite3";
+import { createCanvas } from "canvas";
+
+// Test sharp: create a 1x1 PNG buffer
+const sharpBufferPromise = sharp({
+  create: {
+    width: 1,
+    height: 1,
+    channels: 4,
+    background: { r: 0, g: 0, b: 0, alpha: 0 },
+  },
+})
+  .png()
+  .toBuffer();
+
+// Test sqlite3: open an in-memory database
+const sqliteDb = new sqlite3.Database(":memory:", (err) => {
+  if (err) {
+    console.error("sqlite3 error:", err);
+  } else {
+    console.log("sqlite3 in-memory database opened");
+  }
+});
+
+// Test canvas: create a 100x100 canvas and draw a rectangle
+const canvas = createCanvas(100, 100);
+const ctx = canvas.getContext("2d");
+ctx.fillStyle = "red";
+ctx.fillRect(10, 10, 80, 80);
+
 export const convertVideo = task({
   id: "convert-video",
   retry: {

references/v3-catalog/trigger.config.ts

@@ -3,7 +3,7 @@ import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
 import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
 import { esbuildPlugin } from "@trigger.dev/build";
 import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform";
-import { ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core";
+import { additionalFiles, ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core";
 import { puppeteer } from "@trigger.dev/build/extensions/puppeteer";
 import { playwright } from "@trigger.dev/build/extensions/playwright";
 import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
@@ -15,7 +15,6 @@ export default defineConfig({
   project: "yubjwjsfkxnylobaqvqz",
   machine: "medium-1x",
   instrumentations: [new OpenAIInstrumentation()],
-  additionalFiles: ["wrangler/wrangler.toml"],
   maxDuration: 3600,
   dirs: ["./src/trigger"],
   retries: {
@@ -32,7 +31,11 @@ export default defineConfig({
   logLevel: "info",
   build: {
     conditions: ["react-server"],
+    experimental_autoDetectExternal: true,
     extensions: [
+      additionalFiles({
+        files: ["./wrangler/wrangler.toml"],
+      }),
       ffmpeg(),
       emitDecoratorMetadata(),
       audioWaveform(),
@@ -48,6 +51,7 @@ export default defineConfig({
           org: "triggerdev",
           project: "taskhero-examples-basic",
           authToken: process.env.SENTRY_AUTH_TOKEN,
+          telemetry: false,
         }),
         { placement: "last", target: "deploy" }
       ),