github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

v3: config file is now ts by default and uses named exports (#962)

Browse Source 
* A bunch of changes to the config file:

- No more default export, the config is exported via the `config` named export
- `handleError` is moved to a separate named export
- Added the ability to include additional packages (e.g. wrangler) that aren’t included by default (think CLI usage)
- More reliably install packages in dev
- The config is now built first instead of directly imported, meaning we can now support typescript config files (`trigger.config.ts`)

* Update to trigger.config.ts in `trigger.dev init`
This commit is contained in:
Eric Allam
2024-03-21 10:40:44 +00:00
committed by GitHub
parent 737ff9928c
commit 7b3ffe040b
25 changed files with 481 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/launch.json
Vendored
+1 -1
View File
@@ -33,7 +33,7 @@
"type": "node-terminal",
"request": "launch",
"name": "Debug V3 Dev CLI",
"command": "pnpm exec trigger.dev dev",
"command": "pnpm exec trigger.dev dev --log-level debug",
"cwd": "${workspaceFolder}/references/v3-catalog",
"sourceMaps": true
},
packages/cli-v3/package.json
+1
View File
@@ -110,6 +110,7 @@
"gradient-string": "^2.0.2",
"import-meta-resolve": "^4.0.0",
"ink": "^4.4.1",
"jsonc-parser": "^3.2.1",
"jsonlines": "^0.1.1",
"liquidjs": "^10.9.2",
"mock-fs": "^5.2.0",
packages/cli-v3/src/commands/deploy.ts
+63 -26
View File
@@ -1,7 +1,12 @@
import { intro, log, outro, spinner } from "@clack/prompts";
import { depot } from "@depot/cli";
import { context, trace } from "@opentelemetry/api";
import { ResolvedConfig, detectDependencyVersion, flattenAttributes, recordSpanException } from "@trigger.dev/core/v3";
import {
ResolvedConfig,
detectDependencyVersion,
flattenAttributes,
recordSpanException,
} from "@trigger.dev/core/v3";
import chalk from "chalk";
import { Command, Option as CommandOption } from "commander";
import { Metafile, build } from "esbuild";
@@ -29,7 +34,7 @@ import {
import { readConfig } from "../utilities/configFiles.js";
import { createTempDir, readJSONFile, writeJSONFile } from "../utilities/fileSystem";
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
import { detectPackageNameFromImportPath } from "../utilities/installPackages";
import { detectPackageNameFromImportPath, parsePackageName } from "../utilities/installPackages";
import { logger } from "../utilities/logger.js";
import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
import { login } from "./login";
@@ -63,11 +68,7 @@ export function configureDeployCommand(program: Command) {
"prod"
)
.option("-T, --skip-typecheck", "Whether to skip the pre-build typecheck")
.option(
"-c, --config <config file>",
"The name of the config file, found at [path]",
"trigger.config.mjs"
)
.option("-c, --config <config file>", "The name of the config file, found at [path]")
.option(
"-p, --project-ref <project ref>",
"The project ref. Required if there is no config file."
@@ -134,7 +135,11 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
intro("Deploying project");
const authorization = await login({ embedded: true, defaultApiUrl: options.apiUrl, profile: options.profile });
const authorization = await login({
embedded: true,
defaultApiUrl: options.apiUrl,
profile: options.profile,
});
if (!authorization.ok) {
if (authorization.error === "fetch failed") {
@@ -228,7 +233,8 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
const deploymentSpinner = spinner();
deploymentSpinner.start(`Deploying version ${version}`);
const registryHost = deploymentResponse.data.registryHost ?? options.registry ?? "registry.trigger.dev";
const registryHost =
deploymentResponse.data.registryHost ?? options.registry ?? "registry.trigger.dev";
const buildImage = async () => {
if (options.selfHosted) {
@@ -345,7 +351,8 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
);
} else {
outro(
`Version ${version} deployed with ${taskCount} detected task${taskCount === 1 ? "" : "s"
`Version ${version} deployed with ${taskCount} detected task${
taskCount === 1 ? "" : "s"
} ${deploymentLink}`
);
}
@@ -515,14 +522,14 @@ type BuildAndPushImageOptions = {
type BuildAndPushImageResults =
| {
ok: true;
image: string;
digest?: string;
}
ok: true;
image: string;
digest?: string;
}
| {
ok: false;
error: string;
};
ok: false;
error: string;
};
async function buildAndPushImage(
options: BuildAndPushImageOptions
@@ -774,12 +781,12 @@ async function compileProject(
workerContents = workerContents.replace(
"__IMPORTED_PROJECT_CONFIG__",
`import importedConfig from "${configPath}";`
`import * as importedConfigExports from "${configPath}"; const importedConfig = importedConfigExports.config; const handleError = importedConfigExports.handleError;`
);
} else {
workerContents = workerContents.replace(
"__IMPORTED_PROJECT_CONFIG__",
`const importedConfig = undefined;`
`const importedConfig = undefined; const handleError = undefined;`
);
}
@@ -919,8 +926,7 @@ async function compileProject(
// Get all the required dependencies from the metaOutputs and save them to /tmp/dir/package.json
const allImports = [...metaOutput.imports, ...entryPointMetaOutput.imports];
const projectPackageJson = await readJSONFile(join(config.projectDir, "package.json"));
const dependencies = gatherRequiredDependencies(allImports, projectPackageJson);
const dependencies = await gatherRequiredDependencies(allImports, config);
const packageJsonContents = {
name: "trigger-worker",
@@ -1141,10 +1147,12 @@ async function typecheckProject(config: ResolvedConfig, options: DeployCommandOp
// Returns the dependencies that are required by the output that are found in output and the CLI package dependencies
// Returns the dependency names and the version to use (taken from the CLI deps package.json)
function gatherRequiredDependencies(
async function gatherRequiredDependencies(
imports: Metafile["outputs"][string]["imports"],
externalPackageJson?: { dependencies: Record<string, string> }
config: ResolvedConfig
) {
const externalPackageJson = await readJSONFile(join(config.projectDir, "package.json"));
const dependencies: Record<string, string> = {};
for (const file of imports) {
@@ -1165,15 +1173,44 @@ function gatherRequiredDependencies(
continue;
}
const internalDependencyVersion = (packageJson.dependencies as Record<string, string>)[
packageName
] ?? detectDependencyVersion(packageName);
const internalDependencyVersion =
(packageJson.dependencies as Record<string, string>)[packageName] ??
detectDependencyVersion(packageName);
if (internalDependencyVersion) {
dependencies[packageName] = internalDependencyVersion;
}
}
if (config.additionalPackages) {
for (const packageName of config.additionalPackages) {
if (dependencies[packageName]) {
continue;
}
const packageParts = parsePackageName(packageName);
if (packageParts.version) {
dependencies[packageParts.name] = packageParts.version;
continue;
} else {
const externalDependencyVersion = {
...externalPackageJson?.devDependencies,
...externalPackageJson?.dependencies,
}[packageName];
if (externalDependencyVersion) {
dependencies[packageParts.name] = externalDependencyVersion;
continue;
} else {
logger.warn(
`Could not find version for package ${packageName}, add a version specifier to the package name (e.g. ${packageParts.name}@latest) or add it to your project's package.json`
);
}
}
}
}
// Make sure we sort the dependencies by key to ensure consistent hashing
return Object.fromEntries(Object.entries(dependencies).sort(([a], [b]) => a.localeCompare(b)));
}
packages/cli-v3/src/commands/dev.tsx
+70 -42
View File
@@ -27,8 +27,9 @@ import * as packageJson from "../../package.json";
import { CliApiClient } from "../apiClient";
import { CommonCommandOptions, commonOptions, wrapCommandAction } from "../cli/common.js";
import { readConfig } from "../utilities/configFiles";
import { readJSONFile } from "../utilities/fileSystem";
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
import { detectPackageNameFromImportPath } from "../utilities/installPackages";
import { detectPackageNameFromImportPath, parsePackageName } from "../utilities/installPackages";
import { logger } from "../utilities/logger.js";
import { isLoggedIn } from "../utilities/session.js";
import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
@@ -52,11 +53,7 @@ export function configureDevCommand(program: Command) {
.command("dev")
.description("Run your Trigger.dev tasks locally")
.argument("[path]", "The path to the project", ".")
.option(
"-c, --config <config file>",
"The name of the config file, found at [path]",
"trigger.config.mjs"
)
.option("-c, --config <config file>", "The name of the config file, found at [path]")
.option(
"-p, --project-ref <project ref>",
"The project ref. Required if there is no config file."
@@ -83,16 +80,9 @@ export async function devCommand(dir: string, options: DevCommandOptions) {
return;
}
let watcher;
try {
const devInstance = await startDev(dir, options, authorization.auth);
watcher = devInstance.watcher;
const { waitUntilExit } = devInstance.devReactElement;
await waitUntilExit();
} finally {
await watcher?.close();
}
const devInstance = await startDev(dir, options, authorization.auth);
const { waitUntilExit } = devInstance.devReactElement;
await waitUntilExit();
}
async function startDev(
@@ -100,7 +90,6 @@ async function startDev(
options: DevCommandOptions,
authorization: { apiUrl: string; accessToken: string }
) {
let watcher: ReturnType<typeof watch> | undefined;
let rerender: (node: React.ReactNode) => void | undefined;
try {
@@ -110,6 +99,8 @@ async function startDev(
await printStandloneInitialBanner(true);
logger.debug("Starting dev session", { dir, options, authorization });
let config = await readConfig(dir, {
projectRef: options.projectRef,
configFile: options.config,
@@ -117,23 +108,6 @@ async function startDev(
logger.debug("Initial config", { config });
if (config.status === "file") {
watcher = watch(config.path, {
persistent: true,
}).on("change", async (_event) => {
config = await readConfig(dir, { configFile: options.config });
if (config.status === "file") {
logger.log(`${basename(config.path)} changed...`);
logger.debug("New config", { config: config.config });
rerender(await getDevReactElement(config.config, authorization, config.path));
} else {
logger.debug("New config", { config: config.config });
rerender(await getDevReactElement(config.config, authorization));
}
});
}
async function getDevReactElement(
configParam: ResolvedConfig,
authorization: { apiUrl: string; accessToken: string },
@@ -181,14 +155,11 @@ async function startDev(
return {
devReactElement,
watcher,
stop: async () => {
devReactElement.unmount();
await watcher?.close();
},
};
} catch (e) {
await watcher?.close();
throw e;
}
}
@@ -341,12 +312,12 @@ function useDev({
entryPointContents = entryPointContents.replace(
"__IMPORTED_PROJECT_CONFIG__",
`import importedConfig from "${configPath}";`
`import * as importedConfigExports from "${configPath}"; const importedConfig = importedConfigExports.config; const handleError = importedConfigExports.handleError;`
);
} else {
entryPointContents = entryPointContents.replace(
"__IMPORTED_PROJECT_CONFIG__",
`const importedConfig = undefined;`
`const importedConfig = undefined; const handleError = undefined;`
);
}
@@ -366,7 +337,7 @@ function useDev({
minify: false,
sourcemap: "external", // does not set the //# sourceMappingURL= comment in the file, we handle it ourselves
packages: "external", // https://esbuild.github.io/api/#packages
logLevel: "warning",
logLevel: "error",
platform: "node",
format: "cjs", // This is needed to support opentelemetry instrumentation that uses module patching
target: ["node18", "es2020"],
@@ -437,7 +408,7 @@ function useDev({
logger.debug(`Wrote background worker to ${fullPath}`);
const dependencies = gatherRequiredDependencies(metaOutput);
const dependencies = await gatherRequiredDependencies(metaOutput, config);
if (sourceMapFile) {
const sourceMapPath = `${fullPath}.map`;
@@ -447,10 +418,13 @@ function useDev({
const environmentVariablesResponse =
await environmentClient.getEnvironmentVariables(config.project);
const processEnv = gatherProcessEnv();
const backgroundWorker = new BackgroundWorker(fullPath, {
projectConfig: config,
dependencies,
env: {
...processEnv,
TRIGGER_API_URL: apiUrl,
TRIGGER_SECRET_KEY: apiKey,
...(environmentVariablesResponse.success
@@ -678,7 +652,10 @@ function WebsocketFactory(apiKey: string) {
// Returns the dependencies that are required by the output that are found in output and the CLI package dependencies
// Returns the dependency names and the version to use (taken from the CLI deps package.json)
function gatherRequiredDependencies(outputMeta: Metafile["outputs"][string]) {
async function gatherRequiredDependencies(
outputMeta: Metafile["outputs"][string],
config: ResolvedConfig
) {
const dependencies: Record<string, string> = {};
for (const file of outputMeta.imports) {
@@ -701,5 +678,56 @@ function gatherRequiredDependencies(outputMeta: Metafile["outputs"][string]) {
}
}
if (config.additionalPackages) {
const projectPackageJson = await readJSONFile(join(config.projectDir, "package.json"));
for (const packageName of config.additionalPackages) {
if (dependencies[packageName]) {
continue;
}
const packageParts = parsePackageName(packageName);
if (packageParts.version) {
dependencies[packageParts.name] = packageParts.version;
continue;
} else {
const externalDependencyVersion = {
...projectPackageJson?.devDependencies,
...projectPackageJson?.dependencies,
}[packageName];
if (externalDependencyVersion) {
dependencies[packageParts.name] = externalDependencyVersion;
continue;
} else {
logger.warn(
`Could not find version for package ${packageName}, add a version specifier to the package name (e.g. ${packageParts.name}@latest) or add it to your project's package.json`
);
}
}
}
}
return dependencies;
}
function gatherProcessEnv() {
const env = {
NODE_ENV: process.env.NODE_ENV ?? "development",
PATH: process.env.PATH,
USER: process.env.USER,
SHELL: process.env.SHELL,
NVM_INC: process.env.NVM_INC,
NVM_DIR: process.env.NVM_DIR,
NVM_BIN: process.env.NVM_BIN,
LANG: process.env.LANG,
TERM: process.env.TERM,
NODE_PATH: process.env.NODE_PATH,
HOME: process.env.HOME,
BUN_INSTALL: process.env.BUN_INSTALL,
};
// Filter out undefined values
return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined));
}
packages/cli-v3/src/commands/init.ts
+111 -4
View File
@@ -8,6 +8,8 @@ import {
import chalk from "chalk";
import { Command } from "commander";
import { execa } from "execa";
import { applyEdits, modify } from "jsonc-parser";
import { writeFile } from "node:fs/promises";
import { join, relative, resolve } from "node:path";
import terminalLink from "terminal-link";
import { z } from "zod";
@@ -24,7 +26,7 @@ import {
} from "../cli/common.js";
import { readConfig } from "../utilities/configFiles.js";
import { createFileFromTemplate } from "../utilities/createFileFromTemplate";
import { createFile, pathExists } from "../utilities/fileSystem";
import { createFile, pathExists, readFile } from "../utilities/fileSystem";
import { getUserPackageManager } from "../utilities/getUserPackageManager";
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
import { logger } from "../utilities/logger";
@@ -80,7 +82,11 @@ async function _initCommand(dir: string, options: InitCommandOptions) {
intro("Initializing project");
const authorization = await login({ embedded: true, defaultApiUrl: options.apiUrl, profile: options.profile });
const authorization = await login({
embedded: true,
defaultApiUrl: options.apiUrl,
profile: options.profile,
});
if (!authorization.ok) {
if (authorization.error === "fetch failed") {
@@ -145,6 +151,12 @@ async function _initCommand(dir: string, options: InitCommandOptions) {
// Create the trigger dir
await createTriggerDir(dir, options);
// Add trigger.config.ts to tsconfig.json
await addConfigFileToTsConfig(dir, options);
// Ignore .trigger dir
await gitIgnoreDotTriggerDir(dir, options);
const projectDashboard = terminalLink(
"project dashboard",
`${authorization.dashboardUrl}/projects/v3/${selectedProject.externalRef}`
@@ -255,6 +267,101 @@ async function createTriggerDir(dir: string, options: InitCommandOptions) {
});
}
async function gitIgnoreDotTriggerDir(dir: string, options: InitCommandOptions) {
return await tracer.startActiveSpan("gitIgnoreDotTriggerDir", async (span) => {
try {
const projectDir = resolve(process.cwd(), dir);
const gitIgnorePath = join(projectDir, ".gitignore");
span.setAttributes({
"cli.projectDir": projectDir,
"cli.gitIgnorePath": gitIgnorePath,
});
if (!(await pathExists(gitIgnorePath))) {
// Create .gitignore file
await createFile(gitIgnorePath, ".trigger");
log.step(`Added .trigger to .gitignore`);
span.end();
return;
}
// Check if .gitignore already contains .trigger
const gitIgnoreContent = await readFile(gitIgnorePath);
if (gitIgnoreContent.includes(".trigger")) {
span.end();
return;
}
const newGitIgnoreContent = `${gitIgnoreContent}\n.trigger`;
await writeFile(gitIgnorePath, newGitIgnoreContent, "utf-8");
log.step(`Added .trigger to .gitignore`);
span.end();
} catch (e) {
if (!(e instanceof SkipCommandError)) {
recordSpanException(span, e);
}
span.end();
throw e;
}
});
}
async function addConfigFileToTsConfig(dir: string, options: InitCommandOptions) {
return await tracer.startActiveSpan("createTriggerDir", async (span) => {
try {
const projectDir = resolve(process.cwd(), dir);
const tsconfigPath = join(projectDir, "tsconfig.json");
span.setAttributes({
"cli.projectDir": projectDir,
"cli.tsconfigPath": tsconfigPath,
});
const tsconfigContent = await readFile(tsconfigPath);
const edits = modify(tsconfigContent, ["include", -1], "trigger.config.ts", {
isArrayInsertion: true,
formattingOptions: {
tabSize: 2,
insertSpaces: true,
eol: "\n",
},
});
logger.debug("tsconfig.json edits", { edits });
const newTsconfigContent = applyEdits(tsconfigContent, edits);
logger.debug("new tsconfig.json content", { newTsconfigContent });
await writeFile(tsconfigPath, newTsconfigContent, "utf-8");
log.step(`Added trigger.config.ts to tsconfig.json`);
span.end();
} catch (e) {
if (!(e instanceof SkipCommandError)) {
recordSpanException(span, e);
}
span.end();
throw e;
}
});
}
async function installPackages(dir: string, options: InitCommandOptions) {
return await tracer.startActiveSpan("installPackages", async (span) => {
const installSpinner = spinner();
@@ -332,8 +439,8 @@ async function writeConfigFile(
spnnr.start("Creating config file");
const projectDir = resolve(process.cwd(), dir);
const templatePath = resolveInternalFilePath("./templates/trigger.config.mjs");
const outputPath = join(projectDir, "trigger.config.mjs");
const templatePath = resolveInternalFilePath("./templates/trigger.config.ts");
const outputPath = join(projectDir, "trigger.config.ts");
span.setAttributes({
"cli.projectDir": projectDir,
packages/cli-v3/src/consts.ts
+1 -1
View File
@@ -10,4 +10,4 @@ export const PKG_ROOT = path.join(distPath, "../");
export const COMMAND_NAME = "trigger.dev";
export const CLOUD_WEB_URL = "https://cloud.trigger.dev";
export const CLOUD_API_URL = "https://api.trigger.dev";
export const CONFIG_FILES = ["trigger.config.js", "trigger.config.mjs"];
export const CONFIG_FILES = ["trigger.config.ts", "trigger.config.js", "trigger.config.mjs"];
packages/cli-v3/src/templates/trigger.config.mjs → packages/cli-v3/src/templates/trigger.config.ts
+2 -3
View File
@@ -1,7 +1,6 @@
// @ts-check
/** @type {import('@trigger.dev/sdk/v3').Config} */
import type { ProjectConfig } from "@trigger.dev/core/v3";
export default {
export const config: ProjectConfig = {
project: "${projectRef}",
retries: {
enabledInDev: false,
packages/cli-v3/src/utilities/configFiles.ts
+45 -17
View File
@@ -1,14 +1,15 @@
import { Config, ResolvedConfig } from "@trigger.dev/core/v3";
import { findUp } from "find-up";
import { mkdirSync, writeFileSync } from "node:fs";
import path from "node:path";
import path, { join } from "node:path";
import { pathToFileURL } from "node:url";
import xdgAppPaths from "xdg-app-paths";
import { z } from "zod";
import { CLOUD_API_URL, CONFIG_FILES } from "../consts.js";
import { readJSONFileSync } from "./fileSystem.js";
import { createTempDir, readJSONFileSync } from "./fileSystem.js";
import { logger } from "./logger.js";
import { findTriggerDirectories, resolveTriggerDirectories } from "./taskFiles.js";
import { build } from "esbuild";
function getGlobalConfigFolderPath() {
const configDir = xdgAppPaths("trigger").config();
@@ -81,6 +82,12 @@ function writeAuthConfigFile(config: UserAuthConfigFile) {
}
async function getConfigPath(dir: string, fileName?: string): Promise<string | undefined> {
logger.debug("Searching for the config file", {
dir,
fileName,
configFiles: CONFIG_FILES,
});
return await findUp(fileName ? [fileName] : CONFIG_FILES, { cwd: dir });
}
@@ -91,14 +98,14 @@ export type ReadConfigOptions = {
export type ReadConfigResult =
| {
status: "file";
config: ResolvedConfig;
path: string;
}
status: "file";
config: ResolvedConfig;
path: string;
}
| {
status: "in-memory";
config: ResolvedConfig;
};
status: "in-memory";
config: ResolvedConfig;
};
export async function readConfig(
dir: string,
@@ -106,12 +113,6 @@ export async function readConfig(
): Promise<ReadConfigResult> {
const absoluteDir = path.resolve(process.cwd(), dir);
logger.debug("Searching for the config file", {
dir,
options,
absoluteDir,
});
const configPath = await getConfigPath(dir, options?.configFile);
if (!configPath) {
@@ -128,9 +129,36 @@ export async function readConfig(
}
}
const tempDir = await createTempDir();
const builtConfigFilePath = join(tempDir, "config.mjs");
const builtConfigFileHref = pathToFileURL(builtConfigFilePath).href;
logger.debug("Building config file", {
configPath,
builtConfigFileHref,
builtConfigFilePath,
});
// We need to build the path to the config file, and then import it?
await build({
entryPoints: [configPath],
bundle: true,
metafile: true,
minify: false,
write: true,
format: "esm",
platform: "node",
target: ["es2018", "node18"],
outfile: builtConfigFilePath,
logLevel: "silent",
});
// import the config file
const userConfigModule = await import(`${pathToFileURL(configPath).href}?_ts=${Date.now()}`);
const rawConfig = await normalizeConfig(userConfigModule ? userConfigModule.default : {});
const userConfigModule = await import(builtConfigFileHref);
const rawConfig = await normalizeConfig(
userConfigModule ? userConfigModule.config : { project: options?.projectRef }
);
const config = Config.parse(rawConfig);
return {
packages/cli-v3/src/utilities/installPackages.ts
+42 -56
View File
@@ -1,8 +1,7 @@
import semver from "semver";
import { execa } from "execa";
import { logger } from "./logger";
import { join } from "node:path";
import { readJSONFile, writeJSONFile } from "./fileSystem";
import { logger } from "./logger";
export type InstallPackagesOptions = { cwd?: string };
@@ -12,64 +11,13 @@ export async function installPackages(
) {
const cwd = options?.cwd ?? process.cwd();
logger.debug(`Installing packages at ${cwd}:`, { packages });
logger.debug("Installing packages", { packages });
// Make sure the cwd has a package.json file (if not create a barebones one)
try {
await readJSONFile(join(cwd, "package.json"));
} catch (error) {
await writeJSONFile(join(cwd, "package.json"), {
name: "temp",
version: "1.0.0",
description: "",
});
}
// Detect with packages have already been installed at the specified version (use semver to compare)
// and only install the ones that are missing or have a different version
const installablePackages = await Promise.all(
Object.entries(packages).map(async ([name, version]) => {
try {
const latestVersion = await getPackageVersion(join(cwd, "node_modules", name));
if (!latestVersion) {
return { name, version };
}
return semver.satisfies(latestVersion, version) ? undefined : { name, version };
} catch (error) {
return { name, version };
}
})
)
.then((packages) => packages.filter(Boolean))
.then((packages) =>
packages.reduce((acc: Record<string, string>, p) => ({ ...acc, [p!.name]: p!.version }), {})
);
if (Object.keys(installablePackages).length === 0) {
return;
}
logger.debug(`Found installable packages`);
logger.table(
Object.entries(installablePackages).map(([name, version]) => ({ name, version })),
"debug"
);
await setPackageJsonDeps(join(cwd, "package.json"), packages);
const childProcess = execa(
"npm",
[
"install",
...Object.entries(installablePackages).map(([name, version]) => `${name}@${version}`),
"--install-strategy",
"nested",
"--ignore-scripts",
"--no-package-lock",
"--no-audit",
"--no-fund",
"--no-save",
],
["install", "--install-strategy", "nested", "--ignore-scripts", "--no-audit", "--no-fund"],
{
cwd,
stderr: "inherit",
@@ -113,3 +61,41 @@ export function detectPackageNameFromImportPath(path: string): string {
return path.split("/")[0] as string;
}
}
export function parsePackageName(packageSpecifier: string): { name: string; version?: string } {
const parts = packageSpecifier.split("@");
if (parts.length === 1 && typeof parts[0] === "string") {
return { name: parts[0] };
}
if (parts.length === 2 && typeof parts[0] === "string" && typeof parts[1] === "string") {
return { name: parts[0], version: parts[1] };
}
return { name: packageSpecifier };
}
async function setPackageJsonDeps(path: string, deps: Record<string, string>) {
try {
const existingPackageJson = await readJSONFile(path);
const newPackageJson = {
...existingPackageJson,
dependencies: {
...deps,
},
};
await writeJSONFile(path, newPackageJson);
} catch (error) {
const defaultPackageJson = {
name: "temp",
version: "1.0.0",
description: "",
dependencies: deps,
};
await writeJSONFile(path, defaultPackageJson);
}
}
packages/cli-v3/src/workers/dev/backgroundWorker.ts
+12 -9
View File
@@ -167,8 +167,8 @@ export class BackgroundWorkerCoordinator {
!completion.ok && completion.skippedRetrying
? " (retrying skipped)"
: !completion.ok && completion.retry !== undefined
? ` (retrying in ${completion.retry.delay}ms)`
: "";
? ` (retrying in ${completion.retry.delay}ms)`
: "";
const resultText = !completion.ok
? completion.error.type === "INTERNAL_ERROR" &&
@@ -181,8 +181,8 @@ export class BackgroundWorkerCoordinator {
const errorText = !completion.ok
? this.#formatErrorLog(completion.error)
: "retry" in completion
? `retry in ${completion.retry}ms`
: "";
? `retry in ${completion.retry}ms`
: "";
const elapsedText = chalk.dim(`(${elapsed.toFixed(2)}ms)`);
@@ -263,7 +263,7 @@ export class BackgroundWorker {
constructor(
public path: string,
private params: BackgroundWorkerParams
) { }
) {}
close() {
if (this._closed) {
@@ -316,7 +316,7 @@ export class BackgroundWorker {
resolved = true;
child.kill();
reject(new Error("Worker timed out"));
}, 1000);
}, 5000);
child.on("message", async (msg: any) => {
const message = this._handler.parseMessage(msg);
@@ -545,7 +545,8 @@ class TaskRunProcess {
logger.debug("initializing task run process", {
env: this.env,
path: this.path,
})
processEnv: process.env,
});
this._child = fork(this.path, {
stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
@@ -700,7 +701,8 @@ class TaskRunProcess {
}
logger.log(
`[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number
`[${this.metadata.version}][${this._currentExecution.run.id}.${
this._currentExecution.attempt.number
}] ${data.toString()}`
);
}
@@ -717,7 +719,8 @@ class TaskRunProcess {
}
logger.error(
`[${this.metadata.version}][${this._currentExecution.run.id}.${this._currentExecution.attempt.number
`[${this.metadata.version}][${this._currentExecution.run.id}.${
this._currentExecution.attempt.number
}] ${data.toString()}`
);
}
packages/cli-v3/src/workers/dev/worker-facade.ts
+11 -2
View File
@@ -1,4 +1,11 @@
import { Config, ProjectConfig, TaskExecutor, preciseDateOriginNow, type TracingSDK } from "@trigger.dev/core/v3";
import {
Config,
ProjectConfig,
TaskExecutor,
preciseDateOriginNow,
type TracingSDK,
type HandleErrorFunction,
} from "@trigger.dev/core/v3";
import "source-map-support/register.js";
__WORKER_SETUP__;
@@ -7,6 +14,7 @@ declare const __WORKER_SETUP__: unknown;
__IMPORTED_PROJECT_CONFIG__;
declare const __IMPORTED_PROJECT_CONFIG__: unknown;
declare const importedConfig: ProjectConfig | undefined;
declare const handleError: HandleErrorFunction | undefined;
declare const __PROJECT_CONFIG__: Config;
declare const tracingSDK: TracingSDK;
@@ -48,7 +56,7 @@ const otelTaskLogger = new OtelTaskLogger({
logger: otelLogger,
tracer: tracer,
level: "info",
preciseDateOrigin
preciseDateOrigin,
});
logger.setGlobalTaskLogger(otelTaskLogger);
@@ -111,6 +119,7 @@ for (const task of tasks) {
consoleInterceptor,
projectConfig: __PROJECT_CONFIG__,
importedConfig,
handleErrorFn: handleError,
})
);
}
packages/cli-v3/src/workers/prod/entry-point.ts
+17
View File
@@ -54,6 +54,7 @@ class ProdWorker {
this.#backgroundWorker = new ProdBackgroundWorker("worker.js", {
projectConfig: __PROJECT_CONFIG__,
env: {
...gatherProcessEnv(),
TRIGGER_API_URL: this.apiUrl,
TRIGGER_SECRET_KEY: this.apiKey,
OTEL_EXPORTER_OTLP_ENDPOINT:
@@ -513,3 +514,19 @@ class ProdWorker {
const prodWorker = new ProdWorker(HTTP_SERVER_PORT);
prodWorker.start();
function gatherProcessEnv() {
const env = {
NODE_ENV: process.env.NODE_ENV ?? "production",
PATH: process.env.PATH,
USER: process.env.USER,
SHELL: process.env.SHELL,
LANG: process.env.LANG,
TERM: process.env.TERM,
NODE_PATH: process.env.NODE_PATH,
HOME: process.env.HOME,
};
// Filter out undefined values
return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined));
}
packages/cli-v3/src/workers/prod/worker-facade.ts
+4 -1
View File
@@ -7,6 +7,7 @@ import {
ZodIpcConnection,
type TracingSDK,
preciseDateOriginNow,
HandleErrorFunction,
} from "@trigger.dev/core/v3";
import "source-map-support/register.js";
@@ -16,6 +17,7 @@ declare const __WORKER_SETUP__: unknown;
__IMPORTED_PROJECT_CONFIG__;
declare const __IMPORTED_PROJECT_CONFIG__: unknown;
declare const importedConfig: ProjectConfig | undefined;
declare const handleError: HandleErrorFunction | undefined;
declare const __PROJECT_CONFIG__: Config;
declare const tracingSDK: TracingSDK;
@@ -47,7 +49,7 @@ const otelTaskLogger = new OtelTaskLogger({
logger: otelLogger,
tracer: tracer,
level: "info",
preciseDateOrigin
preciseDateOrigin,
});
logger.setGlobalTaskLogger(otelTaskLogger);
@@ -110,6 +112,7 @@ for (const task of tasks) {
consoleInterceptor,
projectConfig: __PROJECT_CONFIG__,
importedConfig,
handleErrorFn: handleError,
})
);
}
packages/core/src/v3/otel/tracingSDK.ts
+12
View File
@@ -104,6 +104,14 @@ export class TracingSDK {
const traceProvider = new NodeTracerProvider({
forceFlushTimeoutMillis: config.forceFlushTimeoutMillis ?? 500,
resource: commonResources,
spanLimits: {
attributeCountLimit: 1000,
attributeValueLengthLimit: 1000,
eventCountLimit: 100,
attributePerEventCountLimit: 100,
linkCountLimit: 10,
attributePerLinkCountLimit: 100,
},
});
const spanExporter = new OTLPTraceExporter({
@@ -128,6 +136,10 @@ export class TracingSDK {
// To start a logger, you first need to initialize the Logger provider.
const loggerProvider = new LoggerProvider({
resource: commonResources,
logRecordLimits: {
attributeCountLimit: 1000,
attributeValueLengthLimit: 1000,
},
});
loggerProvider.addLogRecordProcessor(
packages/core/src/v3/schemas/schemas.ts
+1
View File
@@ -21,6 +21,7 @@ export const Config = z.object({
default: RetryOptions.optional(),
})
.optional(),
additionalPackages: z.string().array().optional(),
});
export type Config = z.infer<typeof Config>;
packages/core/src/v3/types/config.ts
+1 -6
View File
@@ -1,4 +1,3 @@
import { HandleErrorFnParams, HandleErrorResult } from ".";
import { RetryOptions } from "../schemas";
export interface ProjectConfig {
@@ -9,9 +8,5 @@ export interface ProjectConfig {
enabledInDev?: boolean;
default?: RetryOptions;
};
handleError?: (
payload: any,
error: unknown,
params: HandleErrorFnParams<any>
) => HandleErrorResult;
additionalPackages?: string[];
}
packages/core/src/v3/types/index.ts
+13
View File
@@ -48,6 +48,19 @@ export type HandleErrorResult =
| HandleErrorModificationOptions
| Promise<undefined | void | HandleErrorModificationOptions>;
export type HandleErrorArgs = {
ctx: Context;
retry?: RetryOptions;
retryAt?: Date;
retryDelayInMs?: number;
};
export type HandleErrorFunction = (
payload: any,
error: unknown,
params: HandleErrorArgs
) => HandleErrorResult;
export type TaskMetadataWithFunctions = TaskMetadataWithFilePath & {
fns: {
run: (payload: any, params: RunFnParams<any>) => Promise<any>;
packages/core/src/v3/workers/taskExecutor.ts
+10 -5
View File
@@ -10,7 +10,7 @@ import {
TaskRunExecutionRetry,
} from "../schemas";
import { SemanticInternalAttributes } from "../semanticInternalAttributes";
import { ProjectConfig, TaskMetadataWithFunctions } from "../types";
import { HandleErrorFunction, ProjectConfig, TaskMetadataWithFunctions } from "../types";
import { flattenAttributes } from "../utils/flattenAttributes";
import { accessoryAttributes } from "../utils/styleAttributes";
import { calculateNextRetryDelay } from "../utils/retries";
@@ -25,6 +25,7 @@ export type TaskExecutorOptions = {
consoleInterceptor: ConsoleInterceptor;
projectConfig: Config;
importedConfig: ProjectConfig | undefined;
handleErrorFn: HandleErrorFunction | undefined;
};
export class TaskExecutor {
@@ -33,6 +34,7 @@ export class TaskExecutor {
private _consoleInterceptor: ConsoleInterceptor;
private _config: Config;
private _importedConfig: ProjectConfig | undefined;
private _handleErrorFn: HandleErrorFunction | undefined;
constructor(
public task: TaskMetadataWithFunctions,
@@ -43,6 +45,7 @@ export class TaskExecutor {
this._consoleInterceptor = options.consoleInterceptor;
this._config = options.projectConfig;
this._importedConfig = options.importedConfig;
this._handleErrorFn = options.handleErrorFn;
}
async execute(
@@ -224,7 +227,9 @@ export class TaskExecutor {
| { status: "skipped"; error?: unknown } // skipped is different than noop, it means that the task was skipped from retrying, instead of just not retrying
| { status: "noop"; error?: unknown }
> {
const retry = this.task.retry ?? this._config.retries?.default;
const retriesConfig = this._importedConfig?.retries ?? this._config.retries;
const retry = this.task.retry ?? retriesConfig?.default;
if (!retry) {
return { status: "noop" };
@@ -234,8 +239,8 @@ export class TaskExecutor {
if (
execution.environment.type === "DEVELOPMENT" &&
typeof this._config.retries?.enabledInDev === "boolean" &&
!this._config.retries.enabledInDev
typeof retriesConfig?.enabledInDev === "boolean" &&
!retriesConfig.enabledInDev
) {
return { status: "skipped" };
}
@@ -251,7 +256,7 @@ export class TaskExecutor {
retryAt: delay ? new Date(Date.now() + delay) : undefined,
})
: this._importedConfig
? await this._importedConfig.handleError?.(payload, error, {
? await this._handleErrorFn?.(payload, error, {
ctx,
retry,
retryDelayInMs: delay,
pnpm-lock.yaml
Generated
+9 -4
View File
@@ -1078,6 +1078,7 @@ importers:
gradient-string: ^2.0.2
import-meta-resolve: ^4.0.0
ink: ^4.4.1
jsonc-parser: ^3.2.1
jsonlines: ^0.1.1
liquidjs: ^10.9.2
mock-fs: ^5.2.0
@@ -1148,6 +1149,7 @@ importers:
gradient-string: 2.0.2
import-meta-resolve: 4.0.0
ink: 4.4.1_7kh72gklg5qjlh5zc6s6v3p6v4
jsonc-parser: 3.2.1
jsonlines: 0.1.1
liquidjs: 10.9.3
mock-fs: 5.2.0
@@ -3487,7 +3489,7 @@ packages:
resolution: {integrity: sha512-C/BaXcnnvBCmHTpz/VGZ8jgtE2aYlW4hxDhseJAWZb7gqGM/qtCK6iZUb0TyKFf7BOUsBH7Q7fkRsDRhg1XklQ==}
engines: {node: '>=6.9.0'}
dependencies:
'@babel/helper-validator-identifier': 7.22.15
'@babel/helper-validator-identifier': 7.22.20
chalk: 2.4.2
js-tokens: 4.0.0
@@ -28900,6 +28902,9 @@ packages:
/jsonc-parser/3.2.0:
resolution: {integrity: sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==}
/jsonc-parser/3.2.1:
resolution: {integrity: sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==}
/jsonfile/4.0.0:
resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==}
optionalDependencies:
@@ -31830,7 +31835,7 @@ packages:
bl: 5.1.0
chalk: 5.3.0
cli-cursor: 4.0.0
cli-spinners: 2.9.1
cli-spinners: 2.9.2
is-interactive: 2.0.0
is-unicode-supported: 1.3.0
log-symbols: 5.1.0
@@ -32541,7 +32546,7 @@ packages:
/pkg-types/1.0.3:
resolution: {integrity: sha512-nN7pYi0AQqJnoLPC9eHFQ8AcyaixBUOwvqc5TDnIKCMEE6I0y8P7OKA7fPexsXGCGxQDl/cmrLAp26LhcwxZ4A==}
dependencies:
jsonc-parser: 3.2.0
jsonc-parser: 3.2.1
mlly: 1.4.2
pathe: 1.1.1
@@ -37084,7 +37089,7 @@ packages:
dependencies:
bs-logger: 0.2.6
fast-json-stable-stringify: 2.1.0
jest: 29.6.2_@types+node@18.17.1
jest: 29.6.2_@types+node@18.15.13
jest-util: 29.6.2
json5: 2.2.3
lodash.memoize: 4.1.2
references/v3-catalog/.gitignore
+1
View File
@@ -0,0 +1 @@
.trigger
references/v3-catalog/src/handleError.ts
+5
View File
@@ -0,0 +1,5 @@
import type { HandleErrorFunction } from "@trigger.dev/core/v3";
export const handleError: HandleErrorFunction = async (payload, error, { ctx, retry }) => {
console.log("GOT TO handleError FUNCTION");
};
references/v3-catalog/src/trigger/wrangler.ts
+30
View File
@@ -0,0 +1,30 @@
import { logger, task } from "@trigger.dev/sdk/v3";
import { exec } from "node:child_process";
import { join } from "node:path";
const wranglerPath = join(__dirname, "node_modules", ".bin", "wrangler");
export const wranglerTask = task({
id: "wrangler-task",
run: async () => {
logger.log(`Running wrangler from ${wranglerPath}`, {
processEnv: process.env,
cwd: process.cwd(),
});
const version = await new Promise<string>((resolve, reject) => {
exec(`${wranglerPath} --version`, (error, stdout, stderr) => {
if (error) {
reject(error);
return;
}
resolve(stdout.trim());
});
});
return {
version,
};
},
});
references/v3-catalog/trigger.config.mjs
-16
View File
@@ -1,16 +0,0 @@
// @ts-check
/** @type {import('@trigger.dev/sdk/v3').Config} */
export default {
project: "yubjwjsfkxnylobaqvqz",
retries: {
enabledInDev: false,
default: {
maxAttempts: 3,
minTimeoutInMs: 1000,
maxTimeoutInMs: 10000,
factor: 2,
randomize: true,
},
},
};
references/v3-catalog/trigger.config.ts
+18
View File
@@ -0,0 +1,18 @@
import type { ProjectConfig } from "@trigger.dev/core/v3";
export { handleError } from "./src/handleError";
export const config: ProjectConfig = {
project: "yubjwjsfkxnylobaqvqz",
retries: {
enabledInDev: true,
default: {
maxAttempts: 3,
minTimeoutInMs: 1000,
maxTimeoutInMs: 10000,
factor: 2,
randomize: true,
},
},
additionalPackages: ["wrangler@3.35.0"],
};
references/v3-catalog/tsconfig.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"extends": "@trigger.dev/tsconfig/node18.json",
"include": ["./src/**/*.ts", "trigger.config.mjs"],
"include": ["./src/**/*.ts", "trigger.config.ts"],
"compilerOptions": {
"baseUrl": ".",
"lib": ["DOM", "DOM.Iterable"],