v3: pnpm deploy & various fixes for dependency resolution (#1012)

* Don’t swallow some error messages when deploying

* Try and resolve dependency versions using pnpm/npm/yarn

* Adding some debug logs around dependency resolution

* Use npm list instead of npm show to resolve deps

* Improve the dependency resolution

.changeset/cuddly-feet-approve.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Don’t swallow some error messages when deploying

packages/cli-v3/package.json

@@ -38,7 +38,6 @@
     "@trigger.dev/core-apps": "workspace:*",
     "@trigger.dev/tsconfig": "workspace:*",
     "@types/gradient-string": "^1.1.2",
-    "@types/jsonlines": "^0.1.5",
     "@types/mock-fs": "^4.13.1",
     "@types/node": "18",
     "@types/object-hash": "^3.0.6",
@@ -106,7 +105,6 @@
     "import-meta-resolve": "^4.0.0",
     "ink": "^4.4.1",
     "jsonc-parser": "^3.2.1",
-    "jsonlines": "^0.1.1",
     "liquidjs": "^10.9.2",
     "mock-fs": "^5.2.0",
     "nanoid": "^4.0.2",
@@ -135,4 +133,4 @@
   "engines": {
     "node": ">=18.0.0"
   }
-}
+}

packages/cli-v3/src/commands/deploy.ts

@@ -55,6 +55,7 @@ import {
   parseNpmInstallError,
 } from "../utilities/deployErrors";
 import { safeJsonParse } from "../utilities/safeJsonParse";
+import { JavascriptProject } from "../utilities/javascriptProject";
 
 const DeployCommandOptions = CommonCommandOptions.extend({
   skipTypecheck: z.boolean().default(false),
@@ -423,7 +424,8 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
         }
 
         const parsedError = finishedDeployment.errorData.stack
-          ? parseBuildErrorStack(finishedDeployment.errorData)
+          ? parseBuildErrorStack(finishedDeployment.errorData) ??
+            finishedDeployment.errorData.message
           : finishedDeployment.errorData.message;
 
         if (typeof parsedError === "string") {
@@ -1118,13 +1120,9 @@ async function compileProject(
       // Get all the required dependencies from the metaOutputs and save them to /tmp/dir/package.json
       const allImports = [...metaOutput.imports, ...entryPointMetaOutput.imports];
 
-      const externalPackageJson = await readJSONFile(join(config.projectDir, "package.json"));
+      const javascriptProject = new JavascriptProject(config.projectDir);
 
-      const dependencies = await gatherRequiredDependencies(
-        allImports,
-        config,
-        externalPackageJson
-      );
+      const dependencies = await gatherRequiredDependencies(allImports, config, javascriptProject);
 
       const packageJsonContents = {
         name: "trigger-worker",
@@ -1132,7 +1130,7 @@ async function compileProject(
         description: "",
         dependencies,
         scripts: {
-          postinstall: externalPackageJson?.scripts?.postinstall,
+          ...javascriptProject.scripts,
         },
       };
 
@@ -1377,7 +1375,7 @@ async function typecheckProject(config: ResolvedConfig, options: DeployCommandOp
 async function gatherRequiredDependencies(
   imports: Metafile["outputs"][string]["imports"],
   config: ResolvedConfig,
-  projectPackageJson: any
+  project: JavascriptProject
 ) {
   const dependencies: Record<string, string> = {};
 
@@ -1392,7 +1390,7 @@ async function gatherRequiredDependencies(
       continue;
     }
 
-    const externalDependencyVersion = (projectPackageJson?.dependencies ?? {})[packageName];
+    const externalDependencyVersion = await project.resolve(packageName);
 
     if (externalDependencyVersion) {
       dependencies[packageName] = stripWorkspaceFromVersion(externalDependencyVersion);
@@ -1420,10 +1418,9 @@ async function gatherRequiredDependencies(
         dependencies[packageParts.name] = packageParts.version;
         continue;
       } else {
-        const externalDependencyVersion = {
-          ...projectPackageJson?.devDependencies,
-          ...projectPackageJson?.dependencies,
-        }[packageName];
+        const externalDependencyVersion = await project.resolve(packageParts.name, {
+          allowDev: true,
+        });
 
         if (externalDependencyVersion) {
           dependencies[packageParts.name] = externalDependencyVersion;

packages/cli-v3/src/commands/dev.tsx

@@ -561,7 +561,7 @@ function useDev({
                   } else if (e instanceof UncaughtExceptionError) {
                     const parsedBuildError = parseBuildErrorStack(e.originalError);
 
-                    if (typeof parsedBuildError !== "string") {
+                    if (parsedBuildError && typeof parsedBuildError !== "string") {
                       logESMRequireError(
                         parsedBuildError,
                         configPath

packages/cli-v3/src/utilities/deployErrors.ts

@@ -20,7 +20,7 @@ function errorIsErrorLike(error: unknown): error is Error {
   );
 }
 
-export function parseBuildErrorStack(error: unknown): BuildError {
+export function parseBuildErrorStack(error: unknown): BuildError | undefined {
   if (typeof error === "string") {
     return error;
   }
@@ -48,8 +48,6 @@ export function parseBuildErrorStack(error: unknown): BuildError {
       return error.message;
     }
   }
-
-  return "Unknown error";
 }
 
 export function logESMRequireError(parsedError: ESMRequireError, resolvedConfig: ReadConfigResult) {

packages/cli-v3/src/utilities/getUserPackageManager.spec.ts

@@ -1,114 +0,0 @@
-import { randomUUID } from "crypto";
-import { pathExists } from "./fileSystem.js";
-import { getUserPackageManager } from "./getUserPackageManager.js";
-import * as pathModule from "path";
-import { Mock } from "vitest";
-
-vi.mock("path", () => {
-  const path = {
-    join: vi.fn().mockImplementation((...paths: string[]) => paths.join("/")),
-  };
-
-  return {
-    ...path,
-    default: path,
-  };
-});
-
-vi.mock("./fileSystem.ts", () => ({
-  pathExists: vi.fn().mockResolvedValue(false),
-}));
-
-describe(getUserPackageManager.name, () => {
-  let path: string;
-
-  beforeEach(() => {
-    path = randomUUID();
-  });
-
-  afterEach(() => {
-    vi.clearAllMocks();
-  });
-
-  afterAll(() => {
-    vi.restoreAllMocks();
-  });
-
-  describe(`should use ${pathExists.name} to check for package manager artifacts`, () => {
-    it("should join the path with the artifact name", async () => {
-      await getUserPackageManager(path);
-
-      expect(pathModule.join).toBeCalledWith(path, "yarn.lock");
-      expect(pathModule.join).toBeCalledWith(path, "pnpm-lock.yaml");
-      expect(pathModule.join).toBeCalledWith(path, "package-lock.json");
-    });
-
-    it(`should call ${pathExists.name} with the path.join result`, async () => {
-      const expected = randomUUID();
-
-      (pathModule.join as Mock).mockReturnValueOnce(expected);
-
-      await getUserPackageManager(path);
-
-      expect(pathExists).toBeCalledWith(expected);
-    });
-
-    it('should return "yarn" if yarn.lock exists', async () => {
-      (pathExists as Mock).mockImplementation((path: string) => path.endsWith("yarn.lock"));
-
-      expect(await getUserPackageManager(path)).toBe("yarn");
-    });
-
-    it('should return "pnpm" if pnpm-lock.yaml exists', async () => {
-      (pathExists as Mock).mockImplementation(async (path: string) =>
-        path.endsWith("pnpm-lock.yaml")
-      );
-
-      expect(await getUserPackageManager(path)).toBe("pnpm");
-    });
-
-    it('should return "npm" if package-lock.json exists', async () => {
-      (pathExists as Mock).mockImplementation((path: string) => path.endsWith("package-lock.json"));
-
-      expect(await getUserPackageManager(path)).toBe("npm");
-    });
-
-    it('should return "npm" if npm-shrinkwrap.json exists', async () => {
-      (pathExists as Mock).mockImplementation((path: string) =>
-        path.endsWith("npm-shrinkwrap.json")
-      );
-
-      expect(await getUserPackageManager(path)).toBe("npm");
-    });
-  });
-
-  describe(`if doesn't found artifacts, should use process.env.npm_config_user_agent to detect package manager`, () => {
-    beforeEach(() => {
-      (pathExists as Mock).mockResolvedValue(false);
-    });
-
-    it('should return "yarn" if process.env.npm_config_user_agent starts with "yarn"', async () => {
-      process.env.npm_config_user_agent = "yarn";
-
-      expect(await getUserPackageManager(path)).toBe("yarn");
-    });
-
-    it('should return "pnpm" if process.env.npm_config_user_agent starts with "pnpm"', async () => {
-      process.env.npm_config_user_agent = "pnpm";
-
-      expect(await getUserPackageManager(path)).toBe("pnpm");
-    });
-
-    it('if doesn\'t start with "yarn" or "pnpm", should return "npm"', async () => {
-      process.env.npm_config_user_agent = randomUUID();
-
-      expect(await getUserPackageManager(path)).toBe("npm");
-    });
-
-    it('should return "npm" if process.env.npm_config_user_agent is not set', async () => {
-      delete process.env.npm_config_user_agent;
-
-      expect(await getUserPackageManager(path)).toBe("npm");
-    });
-  });
-});

packages/cli-v3/src/utilities/getUserPackageManager.ts

@@ -1,5 +1,4 @@
-import pathModule from "path";
-import { pathExists } from "./fileSystem.js";
+import { findUp } from "find-up";
 
 export type PackageManager = "npm" | "pnpm" | "yarn";
 
@@ -38,8 +37,8 @@ async function detectPackageManagerFromArtifacts(path: string): Promise<PackageM
   ];
 
   for (const { name, pm } of packageFiles) {
-    const exists = await pathExists(pathModule.join(path, name));
-    if (exists) {
+    const foundPath = await findUp(name, { cwd: path });
+    if (typeof foundPath === "string") {
       return pm;
     }
   }

packages/cli-v3/src/utilities/javascriptProject.ts

@@ -0,0 +1,250 @@
+import { $ } from "execa";
+import { join } from "node:path";
+import { readJSONFileSync } from "./fileSystem";
+import { logger } from "./logger";
+import { PackageManager, getUserPackageManager } from "./getUserPackageManager";
+
+export type ResolveOptions = { allowDev: boolean };
+
+const BuiltInModules = new Set([
+  "assert",
+  "async_hooks",
+  "buffer",
+  "child_process",
+  "cluster",
+  "console",
+  "constants",
+  "crypto",
+  "dgram",
+  "dns",
+  "domain",
+  "events",
+  "fs",
+  "http",
+  "http2",
+  "https",
+  "inspector",
+  "module",
+  "net",
+  "os",
+  "path",
+  "perf_hooks",
+  "process",
+  "punycode",
+  "querystring",
+  "readline",
+  "repl",
+  "stream",
+  "string_decoder",
+  "timers",
+  "tls",
+  "trace_events",
+  "tty",
+  "url",
+  "util",
+  "v8",
+  "vm",
+  "worker_threads",
+  "zlib",
+]);
+
+export class JavascriptProject {
+  private _packageJson?: any;
+  private _packageManager?: PackageManager;
+
+  constructor(private projectPath: string) {}
+
+  private get packageJson() {
+    if (!this._packageJson) {
+      this._packageJson = readJSONFileSync(join(this.projectPath, "package.json"));
+    }
+
+    return this._packageJson;
+  }
+
+  public get scripts(): Record<string, string> {
+    return {
+      postinstall: this.packageJson.scripts?.postinstall,
+    };
+  }
+
+  async resolve(packageName: string, options?: ResolveOptions): Promise<string | undefined> {
+    if (BuiltInModules.has(packageName)) {
+      return undefined;
+    }
+
+    if (!this._packageManager) {
+      this._packageManager = await getUserPackageManager(this.projectPath);
+    }
+
+    const packageManager = this._packageManager;
+
+    const opts = { allowDev: false, ...options };
+
+    const packageJsonVersion = this.packageJson.dependencies?.[packageName];
+
+    if (typeof packageJsonVersion === "string") {
+      return packageJsonVersion;
+    }
+
+    if (opts.allowDev) {
+      const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
+
+      if (typeof devPackageJsonVersion === "string") {
+        return devPackageJsonVersion;
+      }
+    }
+
+    const command =
+      packageManager === "npm"
+        ? new NPMCommands()
+        : packageManager === "pnpm"
+        ? new PNPMCommands()
+        : new YarnCommands();
+
+    try {
+      const version = await command.resolveDependencyVersion(packageName, {
+        cwd: this.projectPath,
+      });
+
+      if (version) {
+        return version;
+      }
+    } catch (error) {
+      logger.debug(`Failed to resolve dependency version using ${command.name}`, {
+        packageName,
+        error,
+      });
+    }
+  }
+}
+
+type PnpmList = {
+  name: string;
+  path: string;
+  version: string;
+  private: boolean;
+  dependencies?: Record<
+    string,
+    {
+      from: string;
+      version: string;
+      resolved: string;
+      path: string;
+    }
+  >;
+}[];
+
+type PackageManagerOptions = {
+  cwd?: string;
+};
+
+interface PackageManagerCommands {
+  resolveDependencyVersion(
+    packageName: string,
+    options: PackageManagerOptions
+  ): Promise<string | undefined>;
+}
+
+class PNPMCommands implements PackageManagerCommands {
+  get name() {
+    return "pnpm";
+  }
+
+  async resolveDependencyVersion(
+    packageName: string,
+    options: PackageManagerOptions
+  ): Promise<string | undefined> {
+    const cmd = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
+    const { stdout } = await $({ cwd: options.cwd })`${cmd} list ${packageName} -r --json`;
+    const result = JSON.parse(stdout) as PnpmList;
+
+    logger.debug(`Resolving ${packageName} version using pnpm`, { result });
+
+    // Return the first dependency version that matches the package name
+    for (const dep of result) {
+      const dependency = dep.dependencies?.[packageName];
+
+      if (dependency) {
+        return dependency.version;
+      }
+    }
+  }
+}
+
+type NpmDependency = {
+  version: string;
+  resolved: string;
+  overridden: boolean;
+  required?: { version: string };
+  dependencies?: Record<string, NpmDependency>;
+};
+
+type NpmListOutput = {
+  dependencies: Record<string, NpmDependency>;
+};
+
+class NPMCommands implements PackageManagerCommands {
+  get name() {
+    return "npm";
+  }
+
+  async resolveDependencyVersion(
+    packageName: string,
+    options: PackageManagerOptions
+  ): Promise<string | undefined> {
+    const cmd = process.platform === "win32" ? "npm.cmd" : "npm";
+    const { stdout } = await $({ cwd: options.cwd })`${cmd} list ${packageName} --json`;
+    const output = JSON.parse(stdout) as NpmListOutput;
+
+    logger.debug(`Resolving ${packageName} version using npm`, { output });
+
+    return this.#recursivelySearchDependencies(output.dependencies, packageName);
+  }
+
+  #recursivelySearchDependencies(
+    dependencies: Record<string, NpmDependency>,
+    packageName: string
+  ): string | undefined {
+    for (const [name, dependency] of Object.entries(dependencies)) {
+      if (name === packageName) {
+        return dependency.version;
+      }
+
+      if (dependency.dependencies) {
+        const result = this.#recursivelySearchDependencies(dependency.dependencies, packageName);
+
+        if (result) {
+          return result;
+        }
+      }
+    }
+  }
+}
+
+class YarnCommands implements PackageManagerCommands {
+  get name() {
+    return "yarn";
+  }
+
+  async resolveDependencyVersion(
+    packageName: string,
+    options: PackageManagerOptions
+  ): Promise<string | undefined> {
+    const cmd = process.platform === "win32" ? "yarn.cmd" : "yarn";
+
+    const { stdout } = await $({ cwd: options.cwd })`${cmd} info ${packageName} --json`;
+
+    const lines = stdout.split("\n");
+
+    logger.debug(`Resolving ${packageName} version using yarn`, { lines });
+
+    for (const line of lines) {
+      const json = JSON.parse(line);
+
+      if (json.value === packageName) {
+        return json.children.Version;
+      }
+    }
+  }
+}

packages/cli-v3/src/utilities/packageManagers.ts

@@ -1,171 +0,0 @@
-import { $ } from "execa";
-import jsonlines from "jsonlines";
-import { getUserPackageManager } from "./getUserPackageManager";
-import { keyValueBy } from "./keyValueBy";
-
-export async function listPackageDependencies(
-  path: string,
-  tag: string | undefined = undefined
-): Promise<Record<string, string | undefined>> {
-  const packageManager = await getPackageManagerCommands(path);
-
-  const list = await packageManager.list({ cwd: path });
-
-  return Object.keys(list).reduce(
-    (acc, dependency) => {
-      const version = list[dependency];
-
-      if (!version) {
-        return acc;
-      }
-
-      if (dependency.startsWith("@trigger.dev/") && version.startsWith("link:")) {
-        acc[dependency] = tag ?? "latest";
-      } else {
-        acc[dependency] = version;
-      }
-
-      return acc;
-    },
-    {} as Record<string, string | undefined>
-  );
-}
-
-type PnpmList = {
-  path: string;
-  private: boolean;
-  dependencies: Record<
-    string,
-    {
-      from: string;
-      version: string;
-      resolved: string;
-    }
-  >;
-}[];
-
-async function getPackageManagerCommands(path: string): Promise<PackageManagerCommands> {
-  const packageManager = await getUserPackageManager(path);
-
-  switch (packageManager) {
-    case "npm":
-      return new NPMCommands();
-    case "pnpm":
-      return new PNPMCommands();
-    case "yarn":
-      return new YarnCommands();
-  }
-}
-
-type ListOptions = {
-  cwd?: string;
-};
-
-interface PackageManagerCommands {
-  list(options: ListOptions): Promise<Record<string, string | undefined>>;
-}
-
-class PNPMCommands implements PackageManagerCommands {
-  async list(options: ListOptions): Promise<Record<string, string | undefined>> {
-    const cmd = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
-    const { stdout } = await $({ cwd: options.cwd })`${cmd} ls --depth 1 --json --long`;
-    const result = JSON.parse(stdout) as PnpmList;
-
-    const list = keyValueBy(result[0]?.dependencies ?? {}, (name, { version }) => ({
-      [name]: version,
-    }));
-
-    return list;
-  }
-}
-
-class NPMCommands implements PackageManagerCommands {
-  async list(options: ListOptions): Promise<Record<string, string | undefined>> {
-    const cmd = process.platform === "win32" ? "npm.cmd" : "npm";
-
-    const { stdout } = await $({ cwd: options.cwd })`${cmd} ls --depth=0 --json`;
-
-    const dependencies = (
-      JSON.parse(stdout) as {
-        dependencies: Record<string, { version?: string; required?: { version: string } }>;
-      }
-    ).dependencies;
-
-    return keyValueBy(dependencies, (name, info) => ({
-      // unmet peer dependencies have a different structure
-      [name]: info.version || info.required?.version,
-    }));
-  }
-}
-
-interface YarnParsedDep {
-  version: string;
-  from: string;
-  required?: {
-    version: string;
-  };
-}
-
-class YarnCommands implements PackageManagerCommands {
-  async list(options: ListOptions): Promise<Record<string, string | undefined>> {
-    const cmd = process.platform === "win32" ? "yarn.cmd" : "yarn";
-
-    const { stdout } = await $`${cmd} list --depth=0 --json --no-progress`;
-
-    const json: { dependencies: Record<string, YarnParsedDep> } = await this.#parseJsonLines(
-      stdout
-    );
-
-    const keyValues: Record<string, string | undefined> = keyValueBy<
-      YarnParsedDep,
-      string | undefined
-    >(json.dependencies, (name, info): { [key: string]: string | undefined } => ({
-      // unmet peer dependencies have a different structure
-      [name]: info.version || info.required?.version,
-    }));
-
-    return keyValues;
-  }
-
-  /**
-   * Parse JSON lines and throw an informative error on failure.
-   *
-   * Note: although this is similar to the NPM parseJson() function we always return the
-   * same concrete-type here, for now.
-   *
-   * @param result    Output from `yarn list --json` to be parsed
-   */
-  #parseJsonLines(result: string): Promise<{ dependencies: Record<string, YarnParsedDep> }> {
-    return new Promise((resolve, reject) => {
-      const dependencies: Record<string, YarnParsedDep> = {};
-
-      const parser = jsonlines.parse();
-
-      parser.on("data", (d) => {
-        // only parse info data
-        // ignore error info, e.g. "Visit https://yarnpkg.com/en/docs/cli/list for documentation about this command."
-        if (d.type === "info" && !d.data.match(/^Visit/)) {
-          // parse package name and version number from info data, e.g. "nodemon@2.0.4" has binaries
-          const [, pkgName, pkgVersion] = d.data.match(/"(@?.*)@(.*)"/) || [];
-
-          dependencies[pkgName] = {
-            version: pkgVersion,
-            from: pkgName,
-          };
-        } else if (d.type === "error") {
-          reject(new Error(d.data));
-        }
-      });
-
-      parser.on("end", () => {
-        resolve({ dependencies });
-      });
-
-      parser.on("error", reject);
-
-      parser.write(result);
-
-      parser.end();
-    });
-  }
-}

pnpm-lock.yaml

@@ -1537,9 +1537,6 @@ importers:
       jsonc-parser:
         specifier: ^3.2.1
         version: 3.2.1
-      jsonlines:
-        specifier: ^0.1.1
-        version: 0.1.1
       liquidjs:
         specifier: ^10.9.2
         version: 10.9.3
@@ -1622,9 +1619,6 @@ importers:
       '@types/gradient-string':
         specifier: ^1.1.2
         version: 1.1.2
-      '@types/jsonlines':
-        specifier: ^0.1.5
-        version: 0.1.5
       '@types/mock-fs':
         specifier: ^4.13.1
         version: 4.13.1
@@ -13954,12 +13948,6 @@ packages:
   /@types/json5@0.0.30:
     resolution: {integrity: sha512-sqm9g7mHlPY/43fcSNrCYfOeX9zkTTK+euO5E6+CVijSMm5tTjkVdwdqRkY3ljjIAf8679vps5jKUoJBCLsMDA==}
 
-  /@types/jsonlines@0.1.5:
-    resolution: {integrity: sha512-/zOl7I350g4/G6fEW9dktpTrkcKqZDMRkr2SuDla0utgwkUXrm7OFXq2WZT0W9Jl7BYoisGbn1EZsV/Z2F9LGg==}
-    dependencies:
-      '@types/node': 20.11.22
-    dev: true
-
   /@types/jsonwebtoken@9.0.1:
     resolution: {integrity: sha512-c5ltxazpWabia/4UzhIoaDcIza4KViOQhdbjRlfcIGVnsE3c3brkz9Z+F/EeJIECOQP7W7US2hNE930cWWkPiw==}
     dependencies: