diff --git a/.vscode/launch.json b/.vscode/launch.json index 25375c084..01f730f61 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -33,7 +33,7 @@ "type": "node-terminal", "request": "launch", "name": "Debug V3 Dev CLI", - "command": "pnpm exec trigger.dev dev", + "command": "pnpm exec trigger.dev dev --log-level debug", "cwd": "${workspaceFolder}/references/v3-catalog", "sourceMaps": true }, diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index eb5698bfd..8f17bf321 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -110,6 +110,7 @@ "gradient-string": "^2.0.2", "import-meta-resolve": "^4.0.0", "ink": "^4.4.1", + "jsonc-parser": "^3.2.1", "jsonlines": "^0.1.1", "liquidjs": "^10.9.2", "mock-fs": "^5.2.0", diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index e785fd68c..547b4c09a 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -1,7 +1,12 @@ import { intro, log, outro, spinner } from "@clack/prompts"; import { depot } from "@depot/cli"; import { context, trace } from "@opentelemetry/api"; -import { ResolvedConfig, detectDependencyVersion, flattenAttributes, recordSpanException } from "@trigger.dev/core/v3"; +import { + ResolvedConfig, + detectDependencyVersion, + flattenAttributes, + recordSpanException, +} from "@trigger.dev/core/v3"; import chalk from "chalk"; import { Command, Option as CommandOption } from "commander"; import { Metafile, build } from "esbuild"; @@ -29,7 +34,7 @@ import { import { readConfig } from "../utilities/configFiles.js"; import { createTempDir, readJSONFile, writeJSONFile } from "../utilities/fileSystem"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; -import { detectPackageNameFromImportPath } from "../utilities/installPackages"; +import { detectPackageNameFromImportPath, parsePackageName } from "../utilities/installPackages"; import { logger } from "../utilities/logger.js"; import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles"; import { login } from "./login"; @@ -63,11 +68,7 @@ export function configureDeployCommand(program: Command) { "prod" ) .option("-T, --skip-typecheck", "Whether to skip the pre-build typecheck") - .option( - "-c, --config ", - "The name of the config file, found at [path]", - "trigger.config.mjs" - ) + .option("-c, --config ", "The name of the config file, found at [path]") .option( "-p, --project-ref ", "The project ref. Required if there is no config file." @@ -134,7 +135,11 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { intro("Deploying project"); - const authorization = await login({ embedded: true, defaultApiUrl: options.apiUrl, profile: options.profile }); + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + }); if (!authorization.ok) { if (authorization.error === "fetch failed") { @@ -228,7 +233,8 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { const deploymentSpinner = spinner(); deploymentSpinner.start(`Deploying version ${version}`); - const registryHost = deploymentResponse.data.registryHost ?? options.registry ?? "registry.trigger.dev"; + const registryHost = + deploymentResponse.data.registryHost ?? options.registry ?? "registry.trigger.dev"; const buildImage = async () => { if (options.selfHosted) { @@ -345,7 +351,8 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { ); } else { outro( - `Version ${version} deployed with ${taskCount} detected task${taskCount === 1 ? "" : "s" + `Version ${version} deployed with ${taskCount} detected task${ + taskCount === 1 ? "" : "s" } ${deploymentLink}` ); } @@ -515,14 +522,14 @@ type BuildAndPushImageOptions = { type BuildAndPushImageResults = | { - ok: true; - image: string; - digest?: string; - } + ok: true; + image: string; + digest?: string; + } | { - ok: false; - error: string; - }; + ok: false; + error: string; + }; async function buildAndPushImage( options: BuildAndPushImageOptions @@ -774,12 +781,12 @@ async function compileProject( workerContents = workerContents.replace( "__IMPORTED_PROJECT_CONFIG__", - `import importedConfig from "${configPath}";` + `import * as importedConfigExports from "${configPath}"; const importedConfig = importedConfigExports.config; const handleError = importedConfigExports.handleError;` ); } else { workerContents = workerContents.replace( "__IMPORTED_PROJECT_CONFIG__", - `const importedConfig = undefined;` + `const importedConfig = undefined; const handleError = undefined;` ); } @@ -919,8 +926,7 @@ async function compileProject( // Get all the required dependencies from the metaOutputs and save them to /tmp/dir/package.json const allImports = [...metaOutput.imports, ...entryPointMetaOutput.imports]; - const projectPackageJson = await readJSONFile(join(config.projectDir, "package.json")); - const dependencies = gatherRequiredDependencies(allImports, projectPackageJson); + const dependencies = await gatherRequiredDependencies(allImports, config); const packageJsonContents = { name: "trigger-worker", @@ -1141,10 +1147,12 @@ async function typecheckProject(config: ResolvedConfig, options: DeployCommandOp // Returns the dependencies that are required by the output that are found in output and the CLI package dependencies // Returns the dependency names and the version to use (taken from the CLI deps package.json) -function gatherRequiredDependencies( +async function gatherRequiredDependencies( imports: Metafile["outputs"][string]["imports"], - externalPackageJson?: { dependencies: Record } + config: ResolvedConfig ) { + const externalPackageJson = await readJSONFile(join(config.projectDir, "package.json")); + const dependencies: Record = {}; for (const file of imports) { @@ -1165,15 +1173,44 @@ function gatherRequiredDependencies( continue; } - const internalDependencyVersion = (packageJson.dependencies as Record)[ - packageName - ] ?? detectDependencyVersion(packageName); + const internalDependencyVersion = + (packageJson.dependencies as Record)[packageName] ?? + detectDependencyVersion(packageName); if (internalDependencyVersion) { dependencies[packageName] = internalDependencyVersion; } } + if (config.additionalPackages) { + for (const packageName of config.additionalPackages) { + if (dependencies[packageName]) { + continue; + } + + const packageParts = parsePackageName(packageName); + + if (packageParts.version) { + dependencies[packageParts.name] = packageParts.version; + continue; + } else { + const externalDependencyVersion = { + ...externalPackageJson?.devDependencies, + ...externalPackageJson?.dependencies, + }[packageName]; + + if (externalDependencyVersion) { + dependencies[packageParts.name] = externalDependencyVersion; + continue; + } else { + logger.warn( + `Could not find version for package ${packageName}, add a version specifier to the package name (e.g. ${packageParts.name}@latest) or add it to your project's package.json` + ); + } + } + } + } + // Make sure we sort the dependencies by key to ensure consistent hashing return Object.fromEntries(Object.entries(dependencies).sort(([a], [b]) => a.localeCompare(b))); } diff --git a/packages/cli-v3/src/commands/dev.tsx b/packages/cli-v3/src/commands/dev.tsx index c11a077c5..10b630acf 100644 --- a/packages/cli-v3/src/commands/dev.tsx +++ b/packages/cli-v3/src/commands/dev.tsx @@ -27,8 +27,9 @@ import * as packageJson from "../../package.json"; import { CliApiClient } from "../apiClient"; import { CommonCommandOptions, commonOptions, wrapCommandAction } from "../cli/common.js"; import { readConfig } from "../utilities/configFiles"; +import { readJSONFile } from "../utilities/fileSystem"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; -import { detectPackageNameFromImportPath } from "../utilities/installPackages"; +import { detectPackageNameFromImportPath, parsePackageName } from "../utilities/installPackages"; import { logger } from "../utilities/logger.js"; import { isLoggedIn } from "../utilities/session.js"; import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles"; @@ -52,11 +53,7 @@ export function configureDevCommand(program: Command) { .command("dev") .description("Run your Trigger.dev tasks locally") .argument("[path]", "The path to the project", ".") - .option( - "-c, --config ", - "The name of the config file, found at [path]", - "trigger.config.mjs" - ) + .option("-c, --config ", "The name of the config file, found at [path]") .option( "-p, --project-ref ", "The project ref. Required if there is no config file." @@ -83,16 +80,9 @@ export async function devCommand(dir: string, options: DevCommandOptions) { return; } - let watcher; - - try { - const devInstance = await startDev(dir, options, authorization.auth); - watcher = devInstance.watcher; - const { waitUntilExit } = devInstance.devReactElement; - await waitUntilExit(); - } finally { - await watcher?.close(); - } + const devInstance = await startDev(dir, options, authorization.auth); + const { waitUntilExit } = devInstance.devReactElement; + await waitUntilExit(); } async function startDev( @@ -100,7 +90,6 @@ async function startDev( options: DevCommandOptions, authorization: { apiUrl: string; accessToken: string } ) { - let watcher: ReturnType | undefined; let rerender: (node: React.ReactNode) => void | undefined; try { @@ -110,6 +99,8 @@ async function startDev( await printStandloneInitialBanner(true); + logger.debug("Starting dev session", { dir, options, authorization }); + let config = await readConfig(dir, { projectRef: options.projectRef, configFile: options.config, @@ -117,23 +108,6 @@ async function startDev( logger.debug("Initial config", { config }); - if (config.status === "file") { - watcher = watch(config.path, { - persistent: true, - }).on("change", async (_event) => { - config = await readConfig(dir, { configFile: options.config }); - - if (config.status === "file") { - logger.log(`${basename(config.path)} changed...`); - logger.debug("New config", { config: config.config }); - rerender(await getDevReactElement(config.config, authorization, config.path)); - } else { - logger.debug("New config", { config: config.config }); - rerender(await getDevReactElement(config.config, authorization)); - } - }); - } - async function getDevReactElement( configParam: ResolvedConfig, authorization: { apiUrl: string; accessToken: string }, @@ -181,14 +155,11 @@ async function startDev( return { devReactElement, - watcher, stop: async () => { devReactElement.unmount(); - await watcher?.close(); }, }; } catch (e) { - await watcher?.close(); throw e; } } @@ -341,12 +312,12 @@ function useDev({ entryPointContents = entryPointContents.replace( "__IMPORTED_PROJECT_CONFIG__", - `import importedConfig from "${configPath}";` + `import * as importedConfigExports from "${configPath}"; const importedConfig = importedConfigExports.config; const handleError = importedConfigExports.handleError;` ); } else { entryPointContents = entryPointContents.replace( "__IMPORTED_PROJECT_CONFIG__", - `const importedConfig = undefined;` + `const importedConfig = undefined; const handleError = undefined;` ); } @@ -366,7 +337,7 @@ function useDev({ minify: false, sourcemap: "external", // does not set the //# sourceMappingURL= comment in the file, we handle it ourselves packages: "external", // https://esbuild.github.io/api/#packages - logLevel: "warning", + logLevel: "error", platform: "node", format: "cjs", // This is needed to support opentelemetry instrumentation that uses module patching target: ["node18", "es2020"], @@ -437,7 +408,7 @@ function useDev({ logger.debug(`Wrote background worker to ${fullPath}`); - const dependencies = gatherRequiredDependencies(metaOutput); + const dependencies = await gatherRequiredDependencies(metaOutput, config); if (sourceMapFile) { const sourceMapPath = `${fullPath}.map`; @@ -447,10 +418,13 @@ function useDev({ const environmentVariablesResponse = await environmentClient.getEnvironmentVariables(config.project); + const processEnv = gatherProcessEnv(); + const backgroundWorker = new BackgroundWorker(fullPath, { projectConfig: config, dependencies, env: { + ...processEnv, TRIGGER_API_URL: apiUrl, TRIGGER_SECRET_KEY: apiKey, ...(environmentVariablesResponse.success @@ -678,7 +652,10 @@ function WebsocketFactory(apiKey: string) { // Returns the dependencies that are required by the output that are found in output and the CLI package dependencies // Returns the dependency names and the version to use (taken from the CLI deps package.json) -function gatherRequiredDependencies(outputMeta: Metafile["outputs"][string]) { +async function gatherRequiredDependencies( + outputMeta: Metafile["outputs"][string], + config: ResolvedConfig +) { const dependencies: Record = {}; for (const file of outputMeta.imports) { @@ -701,5 +678,56 @@ function gatherRequiredDependencies(outputMeta: Metafile["outputs"][string]) { } } + if (config.additionalPackages) { + const projectPackageJson = await readJSONFile(join(config.projectDir, "package.json")); + + for (const packageName of config.additionalPackages) { + if (dependencies[packageName]) { + continue; + } + + const packageParts = parsePackageName(packageName); + + if (packageParts.version) { + dependencies[packageParts.name] = packageParts.version; + continue; + } else { + const externalDependencyVersion = { + ...projectPackageJson?.devDependencies, + ...projectPackageJson?.dependencies, + }[packageName]; + + if (externalDependencyVersion) { + dependencies[packageParts.name] = externalDependencyVersion; + continue; + } else { + logger.warn( + `Could not find version for package ${packageName}, add a version specifier to the package name (e.g. ${packageParts.name}@latest) or add it to your project's package.json` + ); + } + } + } + } + return dependencies; } + +function gatherProcessEnv() { + const env = { + NODE_ENV: process.env.NODE_ENV ?? "development", + PATH: process.env.PATH, + USER: process.env.USER, + SHELL: process.env.SHELL, + NVM_INC: process.env.NVM_INC, + NVM_DIR: process.env.NVM_DIR, + NVM_BIN: process.env.NVM_BIN, + LANG: process.env.LANG, + TERM: process.env.TERM, + NODE_PATH: process.env.NODE_PATH, + HOME: process.env.HOME, + BUN_INSTALL: process.env.BUN_INSTALL, + }; + + // Filter out undefined values + return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined)); +} diff --git a/packages/cli-v3/src/commands/init.ts b/packages/cli-v3/src/commands/init.ts index 6a8a31c05..03dcfc640 100644 --- a/packages/cli-v3/src/commands/init.ts +++ b/packages/cli-v3/src/commands/init.ts @@ -8,6 +8,8 @@ import { import chalk from "chalk"; import { Command } from "commander"; import { execa } from "execa"; +import { applyEdits, modify } from "jsonc-parser"; +import { writeFile } from "node:fs/promises"; import { join, relative, resolve } from "node:path"; import terminalLink from "terminal-link"; import { z } from "zod"; @@ -24,7 +26,7 @@ import { } from "../cli/common.js"; import { readConfig } from "../utilities/configFiles.js"; import { createFileFromTemplate } from "../utilities/createFileFromTemplate"; -import { createFile, pathExists } from "../utilities/fileSystem"; +import { createFile, pathExists, readFile } from "../utilities/fileSystem"; import { getUserPackageManager } from "../utilities/getUserPackageManager"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; import { logger } from "../utilities/logger"; @@ -80,7 +82,11 @@ async function _initCommand(dir: string, options: InitCommandOptions) { intro("Initializing project"); - const authorization = await login({ embedded: true, defaultApiUrl: options.apiUrl, profile: options.profile }); + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + }); if (!authorization.ok) { if (authorization.error === "fetch failed") { @@ -145,6 +151,12 @@ async function _initCommand(dir: string, options: InitCommandOptions) { // Create the trigger dir await createTriggerDir(dir, options); + // Add trigger.config.ts to tsconfig.json + await addConfigFileToTsConfig(dir, options); + + // Ignore .trigger dir + await gitIgnoreDotTriggerDir(dir, options); + const projectDashboard = terminalLink( "project dashboard", `${authorization.dashboardUrl}/projects/v3/${selectedProject.externalRef}` @@ -255,6 +267,101 @@ async function createTriggerDir(dir: string, options: InitCommandOptions) { }); } +async function gitIgnoreDotTriggerDir(dir: string, options: InitCommandOptions) { + return await tracer.startActiveSpan("gitIgnoreDotTriggerDir", async (span) => { + try { + const projectDir = resolve(process.cwd(), dir); + const gitIgnorePath = join(projectDir, ".gitignore"); + + span.setAttributes({ + "cli.projectDir": projectDir, + "cli.gitIgnorePath": gitIgnorePath, + }); + + if (!(await pathExists(gitIgnorePath))) { + // Create .gitignore file + await createFile(gitIgnorePath, ".trigger"); + + log.step(`Added .trigger to .gitignore`); + + span.end(); + + return; + } + + // Check if .gitignore already contains .trigger + const gitIgnoreContent = await readFile(gitIgnorePath); + + if (gitIgnoreContent.includes(".trigger")) { + span.end(); + + return; + } + + const newGitIgnoreContent = `${gitIgnoreContent}\n.trigger`; + + await writeFile(gitIgnorePath, newGitIgnoreContent, "utf-8"); + + log.step(`Added .trigger to .gitignore`); + + span.end(); + } catch (e) { + if (!(e instanceof SkipCommandError)) { + recordSpanException(span, e); + } + + span.end(); + + throw e; + } + }); +} + +async function addConfigFileToTsConfig(dir: string, options: InitCommandOptions) { + return await tracer.startActiveSpan("createTriggerDir", async (span) => { + try { + const projectDir = resolve(process.cwd(), dir); + const tsconfigPath = join(projectDir, "tsconfig.json"); + + span.setAttributes({ + "cli.projectDir": projectDir, + "cli.tsconfigPath": tsconfigPath, + }); + + const tsconfigContent = await readFile(tsconfigPath); + + const edits = modify(tsconfigContent, ["include", -1], "trigger.config.ts", { + isArrayInsertion: true, + formattingOptions: { + tabSize: 2, + insertSpaces: true, + eol: "\n", + }, + }); + + logger.debug("tsconfig.json edits", { edits }); + + const newTsconfigContent = applyEdits(tsconfigContent, edits); + + logger.debug("new tsconfig.json content", { newTsconfigContent }); + + await writeFile(tsconfigPath, newTsconfigContent, "utf-8"); + + log.step(`Added trigger.config.ts to tsconfig.json`); + + span.end(); + } catch (e) { + if (!(e instanceof SkipCommandError)) { + recordSpanException(span, e); + } + + span.end(); + + throw e; + } + }); +} + async function installPackages(dir: string, options: InitCommandOptions) { return await tracer.startActiveSpan("installPackages", async (span) => { const installSpinner = spinner(); @@ -332,8 +439,8 @@ async function writeConfigFile( spnnr.start("Creating config file"); const projectDir = resolve(process.cwd(), dir); - const templatePath = resolveInternalFilePath("./templates/trigger.config.mjs"); - const outputPath = join(projectDir, "trigger.config.mjs"); + const templatePath = resolveInternalFilePath("./templates/trigger.config.ts"); + const outputPath = join(projectDir, "trigger.config.ts"); span.setAttributes({ "cli.projectDir": projectDir, diff --git a/packages/cli-v3/src/consts.ts b/packages/cli-v3/src/consts.ts index 8bb18de4e..e265a6a9e 100644 --- a/packages/cli-v3/src/consts.ts +++ b/packages/cli-v3/src/consts.ts @@ -10,4 +10,4 @@ export const PKG_ROOT = path.join(distPath, "../"); export const COMMAND_NAME = "trigger.dev"; export const CLOUD_WEB_URL = "https://cloud.trigger.dev"; export const CLOUD_API_URL = "https://api.trigger.dev"; -export const CONFIG_FILES = ["trigger.config.js", "trigger.config.mjs"]; +export const CONFIG_FILES = ["trigger.config.ts", "trigger.config.js", "trigger.config.mjs"]; diff --git a/packages/cli-v3/src/templates/trigger.config.mjs b/packages/cli-v3/src/templates/trigger.config.ts similarity index 68% rename from packages/cli-v3/src/templates/trigger.config.mjs rename to packages/cli-v3/src/templates/trigger.config.ts index 2e988ab6e..c9f2a18ae 100644 --- a/packages/cli-v3/src/templates/trigger.config.mjs +++ b/packages/cli-v3/src/templates/trigger.config.ts @@ -1,7 +1,6 @@ -// @ts-check -/** @type {import('@trigger.dev/sdk/v3').Config} */ +import type { ProjectConfig } from "@trigger.dev/core/v3"; -export default { +export const config: ProjectConfig = { project: "${projectRef}", retries: { enabledInDev: false, diff --git a/packages/cli-v3/src/utilities/configFiles.ts b/packages/cli-v3/src/utilities/configFiles.ts index a2f87f0bb..bdabc3157 100644 --- a/packages/cli-v3/src/utilities/configFiles.ts +++ b/packages/cli-v3/src/utilities/configFiles.ts @@ -1,14 +1,15 @@ import { Config, ResolvedConfig } from "@trigger.dev/core/v3"; import { findUp } from "find-up"; import { mkdirSync, writeFileSync } from "node:fs"; -import path from "node:path"; +import path, { join } from "node:path"; import { pathToFileURL } from "node:url"; import xdgAppPaths from "xdg-app-paths"; import { z } from "zod"; import { CLOUD_API_URL, CONFIG_FILES } from "../consts.js"; -import { readJSONFileSync } from "./fileSystem.js"; +import { createTempDir, readJSONFileSync } from "./fileSystem.js"; import { logger } from "./logger.js"; import { findTriggerDirectories, resolveTriggerDirectories } from "./taskFiles.js"; +import { build } from "esbuild"; function getGlobalConfigFolderPath() { const configDir = xdgAppPaths("trigger").config(); @@ -81,6 +82,12 @@ function writeAuthConfigFile(config: UserAuthConfigFile) { } async function getConfigPath(dir: string, fileName?: string): Promise { + logger.debug("Searching for the config file", { + dir, + fileName, + configFiles: CONFIG_FILES, + }); + return await findUp(fileName ? [fileName] : CONFIG_FILES, { cwd: dir }); } @@ -91,14 +98,14 @@ export type ReadConfigOptions = { export type ReadConfigResult = | { - status: "file"; - config: ResolvedConfig; - path: string; - } + status: "file"; + config: ResolvedConfig; + path: string; + } | { - status: "in-memory"; - config: ResolvedConfig; - }; + status: "in-memory"; + config: ResolvedConfig; + }; export async function readConfig( dir: string, @@ -106,12 +113,6 @@ export async function readConfig( ): Promise { const absoluteDir = path.resolve(process.cwd(), dir); - logger.debug("Searching for the config file", { - dir, - options, - absoluteDir, - }); - const configPath = await getConfigPath(dir, options?.configFile); if (!configPath) { @@ -128,9 +129,36 @@ export async function readConfig( } } + const tempDir = await createTempDir(); + + const builtConfigFilePath = join(tempDir, "config.mjs"); + const builtConfigFileHref = pathToFileURL(builtConfigFilePath).href; + + logger.debug("Building config file", { + configPath, + builtConfigFileHref, + builtConfigFilePath, + }); + + // We need to build the path to the config file, and then import it? + await build({ + entryPoints: [configPath], + bundle: true, + metafile: true, + minify: false, + write: true, + format: "esm", + platform: "node", + target: ["es2018", "node18"], + outfile: builtConfigFilePath, + logLevel: "silent", + }); + // import the config file - const userConfigModule = await import(`${pathToFileURL(configPath).href}?_ts=${Date.now()}`); - const rawConfig = await normalizeConfig(userConfigModule ? userConfigModule.default : {}); + const userConfigModule = await import(builtConfigFileHref); + const rawConfig = await normalizeConfig( + userConfigModule ? userConfigModule.config : { project: options?.projectRef } + ); const config = Config.parse(rawConfig); return { diff --git a/packages/cli-v3/src/utilities/installPackages.ts b/packages/cli-v3/src/utilities/installPackages.ts index fdb678fe4..cfa7566b1 100644 --- a/packages/cli-v3/src/utilities/installPackages.ts +++ b/packages/cli-v3/src/utilities/installPackages.ts @@ -1,8 +1,7 @@ -import semver from "semver"; import { execa } from "execa"; -import { logger } from "./logger"; import { join } from "node:path"; import { readJSONFile, writeJSONFile } from "./fileSystem"; +import { logger } from "./logger"; export type InstallPackagesOptions = { cwd?: string }; @@ -12,64 +11,13 @@ export async function installPackages( ) { const cwd = options?.cwd ?? process.cwd(); - logger.debug(`Installing packages at ${cwd}:`, { packages }); + logger.debug("Installing packages", { packages }); - // Make sure the cwd has a package.json file (if not create a barebones one) - try { - await readJSONFile(join(cwd, "package.json")); - } catch (error) { - await writeJSONFile(join(cwd, "package.json"), { - name: "temp", - version: "1.0.0", - description: "", - }); - } - - // Detect with packages have already been installed at the specified version (use semver to compare) - // and only install the ones that are missing or have a different version - const installablePackages = await Promise.all( - Object.entries(packages).map(async ([name, version]) => { - try { - const latestVersion = await getPackageVersion(join(cwd, "node_modules", name)); - - if (!latestVersion) { - return { name, version }; - } - - return semver.satisfies(latestVersion, version) ? undefined : { name, version }; - } catch (error) { - return { name, version }; - } - }) - ) - .then((packages) => packages.filter(Boolean)) - .then((packages) => - packages.reduce((acc: Record, p) => ({ ...acc, [p!.name]: p!.version }), {}) - ); - - if (Object.keys(installablePackages).length === 0) { - return; - } - - logger.debug(`Found installable packages`); - logger.table( - Object.entries(installablePackages).map(([name, version]) => ({ name, version })), - "debug" - ); + await setPackageJsonDeps(join(cwd, "package.json"), packages); const childProcess = execa( "npm", - [ - "install", - ...Object.entries(installablePackages).map(([name, version]) => `${name}@${version}`), - "--install-strategy", - "nested", - "--ignore-scripts", - "--no-package-lock", - "--no-audit", - "--no-fund", - "--no-save", - ], + ["install", "--install-strategy", "nested", "--ignore-scripts", "--no-audit", "--no-fund"], { cwd, stderr: "inherit", @@ -113,3 +61,41 @@ export function detectPackageNameFromImportPath(path: string): string { return path.split("/")[0] as string; } } + +export function parsePackageName(packageSpecifier: string): { name: string; version?: string } { + const parts = packageSpecifier.split("@"); + + if (parts.length === 1 && typeof parts[0] === "string") { + return { name: parts[0] }; + } + + if (parts.length === 2 && typeof parts[0] === "string" && typeof parts[1] === "string") { + return { name: parts[0], version: parts[1] }; + } + + return { name: packageSpecifier }; +} + +async function setPackageJsonDeps(path: string, deps: Record) { + try { + const existingPackageJson = await readJSONFile(path); + + const newPackageJson = { + ...existingPackageJson, + dependencies: { + ...deps, + }, + }; + + await writeJSONFile(path, newPackageJson); + } catch (error) { + const defaultPackageJson = { + name: "temp", + version: "1.0.0", + description: "", + dependencies: deps, + }; + + await writeJSONFile(path, defaultPackageJson); + } +} diff --git a/packages/cli-v3/src/workers/dev/backgroundWorker.ts b/packages/cli-v3/src/workers/dev/backgroundWorker.ts index 391b162b8..f4034fc24 100644 --- a/packages/cli-v3/src/workers/dev/backgroundWorker.ts +++ b/packages/cli-v3/src/workers/dev/backgroundWorker.ts @@ -167,8 +167,8 @@ export class BackgroundWorkerCoordinator { !completion.ok && completion.skippedRetrying ? " (retrying skipped)" : !completion.ok && completion.retry !== undefined - ? ` (retrying in ${completion.retry.delay}ms)` - : ""; + ? ` (retrying in ${completion.retry.delay}ms)` + : ""; const resultText = !completion.ok ? completion.error.type === "INTERNAL_ERROR" && @@ -181,8 +181,8 @@ export class BackgroundWorkerCoordinator { const errorText = !completion.ok ? this.#formatErrorLog(completion.error) : "retry" in completion - ? `retry in ${completion.retry}ms` - : ""; + ? `retry in ${completion.retry}ms` + : ""; const elapsedText = chalk.dim(`(${elapsed.toFixed(2)}ms)`); @@ -263,7 +263,7 @@ export class BackgroundWorker { constructor( public path: string, private params: BackgroundWorkerParams - ) { } + ) {} close() { if (this._closed) { @@ -316,7 +316,7 @@ export class BackgroundWorker { resolved = true; child.kill(); reject(new Error("Worker timed out")); - }, 1000); + }, 5000); child.on("message", async (msg: any) => { const message = this._handler.parseMessage(msg); @@ -545,7 +545,8 @@ class TaskRunProcess { logger.debug("initializing task run process", { env: this.env, path: this.path, - }) + processEnv: process.env, + }); this._child = fork(this.path, { stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"], @@ -700,7 +701,8 @@ class TaskRunProcess { } logger.log( - `[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number + `[${this.metadata.version}][${this._currentExecution.run.id}.${ + this._currentExecution.attempt.number }] ${data.toString()}` ); } @@ -717,7 +719,8 @@ class TaskRunProcess { } logger.error( - `[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number + `[${this.metadata.version}][${this._currentExecution.run.id}.${ + this._currentExecution.attempt.number }] ${data.toString()}` ); } diff --git a/packages/cli-v3/src/workers/dev/worker-facade.ts b/packages/cli-v3/src/workers/dev/worker-facade.ts index 9498ac17a..0592703c6 100644 --- a/packages/cli-v3/src/workers/dev/worker-facade.ts +++ b/packages/cli-v3/src/workers/dev/worker-facade.ts @@ -1,4 +1,11 @@ -import { Config, ProjectConfig, TaskExecutor, preciseDateOriginNow, type TracingSDK } from "@trigger.dev/core/v3"; +import { + Config, + ProjectConfig, + TaskExecutor, + preciseDateOriginNow, + type TracingSDK, + type HandleErrorFunction, +} from "@trigger.dev/core/v3"; import "source-map-support/register.js"; __WORKER_SETUP__; @@ -7,6 +14,7 @@ declare const __WORKER_SETUP__: unknown; __IMPORTED_PROJECT_CONFIG__; declare const __IMPORTED_PROJECT_CONFIG__: unknown; declare const importedConfig: ProjectConfig | undefined; +declare const handleError: HandleErrorFunction | undefined; declare const __PROJECT_CONFIG__: Config; declare const tracingSDK: TracingSDK; @@ -48,7 +56,7 @@ const otelTaskLogger = new OtelTaskLogger({ logger: otelLogger, tracer: tracer, level: "info", - preciseDateOrigin + preciseDateOrigin, }); logger.setGlobalTaskLogger(otelTaskLogger); @@ -111,6 +119,7 @@ for (const task of tasks) { consoleInterceptor, projectConfig: __PROJECT_CONFIG__, importedConfig, + handleErrorFn: handleError, }) ); } diff --git a/packages/cli-v3/src/workers/prod/entry-point.ts b/packages/cli-v3/src/workers/prod/entry-point.ts index b6c369289..6d5de2ac7 100644 --- a/packages/cli-v3/src/workers/prod/entry-point.ts +++ b/packages/cli-v3/src/workers/prod/entry-point.ts @@ -54,6 +54,7 @@ class ProdWorker { this.#backgroundWorker = new ProdBackgroundWorker("worker.js", { projectConfig: __PROJECT_CONFIG__, env: { + ...gatherProcessEnv(), TRIGGER_API_URL: this.apiUrl, TRIGGER_SECRET_KEY: this.apiKey, OTEL_EXPORTER_OTLP_ENDPOINT: @@ -513,3 +514,19 @@ class ProdWorker { const prodWorker = new ProdWorker(HTTP_SERVER_PORT); prodWorker.start(); + +function gatherProcessEnv() { + const env = { + NODE_ENV: process.env.NODE_ENV ?? "production", + PATH: process.env.PATH, + USER: process.env.USER, + SHELL: process.env.SHELL, + LANG: process.env.LANG, + TERM: process.env.TERM, + NODE_PATH: process.env.NODE_PATH, + HOME: process.env.HOME, + }; + + // Filter out undefined values + return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined)); +} diff --git a/packages/cli-v3/src/workers/prod/worker-facade.ts b/packages/cli-v3/src/workers/prod/worker-facade.ts index ad492e031..07c3fadee 100644 --- a/packages/cli-v3/src/workers/prod/worker-facade.ts +++ b/packages/cli-v3/src/workers/prod/worker-facade.ts @@ -7,6 +7,7 @@ import { ZodIpcConnection, type TracingSDK, preciseDateOriginNow, + HandleErrorFunction, } from "@trigger.dev/core/v3"; import "source-map-support/register.js"; @@ -16,6 +17,7 @@ declare const __WORKER_SETUP__: unknown; __IMPORTED_PROJECT_CONFIG__; declare const __IMPORTED_PROJECT_CONFIG__: unknown; declare const importedConfig: ProjectConfig | undefined; +declare const handleError: HandleErrorFunction | undefined; declare const __PROJECT_CONFIG__: Config; declare const tracingSDK: TracingSDK; @@ -47,7 +49,7 @@ const otelTaskLogger = new OtelTaskLogger({ logger: otelLogger, tracer: tracer, level: "info", - preciseDateOrigin + preciseDateOrigin, }); logger.setGlobalTaskLogger(otelTaskLogger); @@ -110,6 +112,7 @@ for (const task of tasks) { consoleInterceptor, projectConfig: __PROJECT_CONFIG__, importedConfig, + handleErrorFn: handleError, }) ); } diff --git a/packages/core/src/v3/otel/tracingSDK.ts b/packages/core/src/v3/otel/tracingSDK.ts index b0105a91f..09ba3b8e0 100644 --- a/packages/core/src/v3/otel/tracingSDK.ts +++ b/packages/core/src/v3/otel/tracingSDK.ts @@ -104,6 +104,14 @@ export class TracingSDK { const traceProvider = new NodeTracerProvider({ forceFlushTimeoutMillis: config.forceFlushTimeoutMillis ?? 500, resource: commonResources, + spanLimits: { + attributeCountLimit: 1000, + attributeValueLengthLimit: 1000, + eventCountLimit: 100, + attributePerEventCountLimit: 100, + linkCountLimit: 10, + attributePerLinkCountLimit: 100, + }, }); const spanExporter = new OTLPTraceExporter({ @@ -128,6 +136,10 @@ export class TracingSDK { // To start a logger, you first need to initialize the Logger provider. const loggerProvider = new LoggerProvider({ resource: commonResources, + logRecordLimits: { + attributeCountLimit: 1000, + attributeValueLengthLimit: 1000, + }, }); loggerProvider.addLogRecordProcessor( diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index ce243d199..06640caec 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -21,6 +21,7 @@ export const Config = z.object({ default: RetryOptions.optional(), }) .optional(), + additionalPackages: z.string().array().optional(), }); export type Config = z.infer; diff --git a/packages/core/src/v3/types/config.ts b/packages/core/src/v3/types/config.ts index f73829842..f0e0d4a1c 100644 --- a/packages/core/src/v3/types/config.ts +++ b/packages/core/src/v3/types/config.ts @@ -1,4 +1,3 @@ -import { HandleErrorFnParams, HandleErrorResult } from "."; import { RetryOptions } from "../schemas"; export interface ProjectConfig { @@ -9,9 +8,5 @@ export interface ProjectConfig { enabledInDev?: boolean; default?: RetryOptions; }; - handleError?: ( - payload: any, - error: unknown, - params: HandleErrorFnParams - ) => HandleErrorResult; + additionalPackages?: string[]; } diff --git a/packages/core/src/v3/types/index.ts b/packages/core/src/v3/types/index.ts index 35e5a8be6..80a2bad86 100644 --- a/packages/core/src/v3/types/index.ts +++ b/packages/core/src/v3/types/index.ts @@ -48,6 +48,19 @@ export type HandleErrorResult = | HandleErrorModificationOptions | Promise; +export type HandleErrorArgs = { + ctx: Context; + retry?: RetryOptions; + retryAt?: Date; + retryDelayInMs?: number; +}; + +export type HandleErrorFunction = ( + payload: any, + error: unknown, + params: HandleErrorArgs +) => HandleErrorResult; + export type TaskMetadataWithFunctions = TaskMetadataWithFilePath & { fns: { run: (payload: any, params: RunFnParams) => Promise; diff --git a/packages/core/src/v3/workers/taskExecutor.ts b/packages/core/src/v3/workers/taskExecutor.ts index 6a2db3045..a93ada2a4 100644 --- a/packages/core/src/v3/workers/taskExecutor.ts +++ b/packages/core/src/v3/workers/taskExecutor.ts @@ -10,7 +10,7 @@ import { TaskRunExecutionRetry, } from "../schemas"; import { SemanticInternalAttributes } from "../semanticInternalAttributes"; -import { ProjectConfig, TaskMetadataWithFunctions } from "../types"; +import { HandleErrorFunction, ProjectConfig, TaskMetadataWithFunctions } from "../types"; import { flattenAttributes } from "../utils/flattenAttributes"; import { accessoryAttributes } from "../utils/styleAttributes"; import { calculateNextRetryDelay } from "../utils/retries"; @@ -25,6 +25,7 @@ export type TaskExecutorOptions = { consoleInterceptor: ConsoleInterceptor; projectConfig: Config; importedConfig: ProjectConfig | undefined; + handleErrorFn: HandleErrorFunction | undefined; }; export class TaskExecutor { @@ -33,6 +34,7 @@ export class TaskExecutor { private _consoleInterceptor: ConsoleInterceptor; private _config: Config; private _importedConfig: ProjectConfig | undefined; + private _handleErrorFn: HandleErrorFunction | undefined; constructor( public task: TaskMetadataWithFunctions, @@ -43,6 +45,7 @@ export class TaskExecutor { this._consoleInterceptor = options.consoleInterceptor; this._config = options.projectConfig; this._importedConfig = options.importedConfig; + this._handleErrorFn = options.handleErrorFn; } async execute( @@ -224,7 +227,9 @@ export class TaskExecutor { | { status: "skipped"; error?: unknown } // skipped is different than noop, it means that the task was skipped from retrying, instead of just not retrying | { status: "noop"; error?: unknown } > { - const retry = this.task.retry ?? this._config.retries?.default; + const retriesConfig = this._importedConfig?.retries ?? this._config.retries; + + const retry = this.task.retry ?? retriesConfig?.default; if (!retry) { return { status: "noop" }; @@ -234,8 +239,8 @@ export class TaskExecutor { if ( execution.environment.type === "DEVELOPMENT" && - typeof this._config.retries?.enabledInDev === "boolean" && - !this._config.retries.enabledInDev + typeof retriesConfig?.enabledInDev === "boolean" && + !retriesConfig.enabledInDev ) { return { status: "skipped" }; } @@ -251,7 +256,7 @@ export class TaskExecutor { retryAt: delay ? new Date(Date.now() + delay) : undefined, }) : this._importedConfig - ? await this._importedConfig.handleError?.(payload, error, { + ? await this._handleErrorFn?.(payload, error, { ctx, retry, retryDelayInMs: delay, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2fafa4084..d85c3e932 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1078,6 +1078,7 @@ importers: gradient-string: ^2.0.2 import-meta-resolve: ^4.0.0 ink: ^4.4.1 + jsonc-parser: ^3.2.1 jsonlines: ^0.1.1 liquidjs: ^10.9.2 mock-fs: ^5.2.0 @@ -1148,6 +1149,7 @@ importers: gradient-string: 2.0.2 import-meta-resolve: 4.0.0 ink: 4.4.1_7kh72gklg5qjlh5zc6s6v3p6v4 + jsonc-parser: 3.2.1 jsonlines: 0.1.1 liquidjs: 10.9.3 mock-fs: 5.2.0 @@ -3487,7 +3489,7 @@ packages: resolution: {integrity: sha512-C/BaXcnnvBCmHTpz/VGZ8jgtE2aYlW4hxDhseJAWZb7gqGM/qtCK6iZUb0TyKFf7BOUsBH7Q7fkRsDRhg1XklQ==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-validator-identifier': 7.22.15 + '@babel/helper-validator-identifier': 7.22.20 chalk: 2.4.2 js-tokens: 4.0.0 @@ -28900,6 +28902,9 @@ packages: /jsonc-parser/3.2.0: resolution: {integrity: sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==} + /jsonc-parser/3.2.1: + resolution: {integrity: sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==} + /jsonfile/4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} optionalDependencies: @@ -31830,7 +31835,7 @@ packages: bl: 5.1.0 chalk: 5.3.0 cli-cursor: 4.0.0 - cli-spinners: 2.9.1 + cli-spinners: 2.9.2 is-interactive: 2.0.0 is-unicode-supported: 1.3.0 log-symbols: 5.1.0 @@ -32541,7 +32546,7 @@ packages: /pkg-types/1.0.3: resolution: {integrity: sha512-nN7pYi0AQqJnoLPC9eHFQ8AcyaixBUOwvqc5TDnIKCMEE6I0y8P7OKA7fPexsXGCGxQDl/cmrLAp26LhcwxZ4A==} dependencies: - jsonc-parser: 3.2.0 + jsonc-parser: 3.2.1 mlly: 1.4.2 pathe: 1.1.1 @@ -37084,7 +37089,7 @@ packages: dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 - jest: 29.6.2_@types+node@18.17.1 + jest: 29.6.2_@types+node@18.15.13 jest-util: 29.6.2 json5: 2.2.3 lodash.memoize: 4.1.2 diff --git a/references/v3-catalog/.gitignore b/references/v3-catalog/.gitignore new file mode 100644 index 000000000..6524f048d --- /dev/null +++ b/references/v3-catalog/.gitignore @@ -0,0 +1 @@ +.trigger \ No newline at end of file diff --git a/references/v3-catalog/src/handleError.ts b/references/v3-catalog/src/handleError.ts new file mode 100644 index 000000000..ce2d60d68 --- /dev/null +++ b/references/v3-catalog/src/handleError.ts @@ -0,0 +1,5 @@ +import type { HandleErrorFunction } from "@trigger.dev/core/v3"; + +export const handleError: HandleErrorFunction = async (payload, error, { ctx, retry }) => { + console.log("GOT TO handleError FUNCTION"); +}; diff --git a/references/v3-catalog/src/trigger/wrangler.ts b/references/v3-catalog/src/trigger/wrangler.ts new file mode 100644 index 000000000..418aa11da --- /dev/null +++ b/references/v3-catalog/src/trigger/wrangler.ts @@ -0,0 +1,30 @@ +import { logger, task } from "@trigger.dev/sdk/v3"; +import { exec } from "node:child_process"; +import { join } from "node:path"; + +const wranglerPath = join(__dirname, "node_modules", ".bin", "wrangler"); + +export const wranglerTask = task({ + id: "wrangler-task", + run: async () => { + logger.log(`Running wrangler from ${wranglerPath}`, { + processEnv: process.env, + cwd: process.cwd(), + }); + + const version = await new Promise((resolve, reject) => { + exec(`${wranglerPath} --version`, (error, stdout, stderr) => { + if (error) { + reject(error); + return; + } + + resolve(stdout.trim()); + }); + }); + + return { + version, + }; + }, +}); diff --git a/references/v3-catalog/trigger.config.mjs b/references/v3-catalog/trigger.config.mjs deleted file mode 100644 index 2b28b67b6..000000000 --- a/references/v3-catalog/trigger.config.mjs +++ /dev/null @@ -1,16 +0,0 @@ -// @ts-check -/** @type {import('@trigger.dev/sdk/v3').Config} */ - -export default { - project: "yubjwjsfkxnylobaqvqz", - retries: { - enabledInDev: false, - default: { - maxAttempts: 3, - minTimeoutInMs: 1000, - maxTimeoutInMs: 10000, - factor: 2, - randomize: true, - }, - }, -}; diff --git a/references/v3-catalog/trigger.config.ts b/references/v3-catalog/trigger.config.ts new file mode 100644 index 000000000..cf442f15a --- /dev/null +++ b/references/v3-catalog/trigger.config.ts @@ -0,0 +1,18 @@ +import type { ProjectConfig } from "@trigger.dev/core/v3"; + +export { handleError } from "./src/handleError"; + +export const config: ProjectConfig = { + project: "yubjwjsfkxnylobaqvqz", + retries: { + enabledInDev: true, + default: { + maxAttempts: 3, + minTimeoutInMs: 1000, + maxTimeoutInMs: 10000, + factor: 2, + randomize: true, + }, + }, + additionalPackages: ["wrangler@3.35.0"], +}; diff --git a/references/v3-catalog/tsconfig.json b/references/v3-catalog/tsconfig.json index d09e514c3..790d76991 100644 --- a/references/v3-catalog/tsconfig.json +++ b/references/v3-catalog/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "@trigger.dev/tsconfig/node18.json", - "include": ["./src/**/*.ts", "trigger.config.mjs"], + "include": ["./src/**/*.ts", "trigger.config.ts"], "compilerOptions": { "baseUrl": ".", "lib": ["DOM", "DOM.Iterable"],