diff --git a/.vscode/launch.json b/.vscode/launch.json index 8aabbd986..1d312c2ce 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -41,7 +41,7 @@ "type": "node-terminal", "request": "launch", "name": "Debug V3 Deploy CLI", - "command": "pnpm exec triggerdev deploy --self-hosted", + "command": "pnpm exec triggerdev deploy --self-hosted --load-image", "cwd": "${workspaceFolder}/references/v3-catalog", "sourceMaps": true }, diff --git a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts index 60763f970..7891c1191 100644 --- a/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts @@ -1,16 +1,13 @@ import { DeploymentErrorData, ExternalBuildData, - TaskMetadataFailedToParseData, - groupTaskMetadataIssuesByTask, + prepareDeploymentError, } from "@trigger.dev/core/v3"; -import { WorkerDeployment, WorkerDeploymentStatus } from "@trigger.dev/database"; -import { z } from "zod"; +import { WorkerDeployment } from "@trigger.dev/database"; import { PrismaClient, prisma } from "~/db.server"; import { Organization } from "~/models/organization.server"; import { Project } from "~/models/project.server"; import { User } from "~/models/user.server"; -import { safeJsonParse } from "~/utils/json"; import { getUsername } from "~/utils/username"; export type ErrorData = { @@ -164,75 +161,12 @@ export class DeploymentPresenter { return; } - const parsedErrorData = DeploymentErrorData.safeParse(errorData); + const deploymentError = DeploymentErrorData.safeParse(errorData); - if (!parsedErrorData.success) { + if (!deploymentError.success) { return; } - if (parsedErrorData.data.name === "TaskMetadataParseError") { - const errorJson = safeJsonParse(parsedErrorData.data.stack); - - if (errorJson) { - const parsedError = TaskMetadataFailedToParseData.safeParse(errorJson); - - if (parsedError.success) { - return { - name: parsedErrorData.data.name, - message: parsedErrorData.data.message, - stack: createTaskMetadataFailedErrorStack(parsedError.data), - stderr: parsedErrorData.data.stderr, - }; - } else { - return { - name: parsedErrorData.data.name, - message: parsedErrorData.data.message, - stderr: parsedErrorData.data.stderr, - }; - } - } else { - return { - name: parsedErrorData.data.name, - message: parsedErrorData.data.message, - stderr: parsedErrorData.data.stderr, - }; - } - } - - return { - name: parsedErrorData.data.name, - message: parsedErrorData.data.message, - stack: parsedErrorData.data.stack, - stderr: parsedErrorData.data.stderr, - }; + return prepareDeploymentError(deploymentError.data); } } - -function createTaskMetadataFailedErrorStack( - data: z.infer -): string { - const stack = []; - - const groupedIssues = groupTaskMetadataIssuesByTask(data.tasks, data.zodIssues); - - for (const key in groupedIssues) { - const taskWithIssues = groupedIssues[key]; - - if (!taskWithIssues) { - continue; - } - - stack.push("\n"); - stack.push(` ❯ ${taskWithIssues.exportName} in ${taskWithIssues.filePath}`); - - for (const issue of taskWithIssues.issues) { - if (issue.path) { - stack.push(` x ${issue.path} ${issue.message}`); - } else { - stack.push(` x ${issue.message}`); - } - } - } - - return stack.join("\n"); -} diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index 2dc3e8d30..12e0544e0 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -114,6 +114,7 @@ "resolve": "^1.22.8", "signal-exit": "^4.1.0", "source-map-support": "0.5.21", + "std-env": "^3.7.0", "terminal-link": "^3.0.0", "tiny-invariant": "^1.2.0", "tinyexec": "^0.2.0", diff --git a/packages/cli-v3/src/build/bundle.ts b/packages/cli-v3/src/build/bundle.ts index 920f5adf0..3be04ca66 100644 --- a/packages/cli-v3/src/build/bundle.ts +++ b/packages/cli-v3/src/build/bundle.ts @@ -9,11 +9,12 @@ import { deployEntryPoints, devEntryPoints, isConfigEntryPoint, - isExecutorEntryPointForTarget, - isIndexerEntryPointForTarget, + isRunWorkerForTarget, + isIndexWorkerForTarget, isLoaderEntryPoint, - isWorkerEntryPointForTarget, + isRunControllerForTarget, shims, + isIndexControllerForTarget, } from "./packageModules.js"; import { buildPlugins } from "./plugins.js"; @@ -34,9 +35,10 @@ export type BundleResult = { files: TaskFile[]; configPath: string; loaderEntryPoint: string | undefined; - workerEntryPoint: string | undefined; - indexerEntryPoint: string | undefined; - executorEntryPoint: string | undefined; + runWorkerEntryPoint: string | undefined; + runControllerEntryPoint: string | undefined; + indexWorkerEntryPoint: string | undefined; + indexControllerEntryPoint: string | undefined; stop: (() => Promise) | undefined; }; @@ -145,9 +147,10 @@ export async function getBundleResultFromBuild( let configPath: string | undefined; let loaderEntryPoint: string | undefined; - let workerEntryPoint: string | undefined; - let executorEntryPoint: string | undefined; - let indexerEntryPoint: string | undefined; + let runWorkerEntryPoint: string | undefined; + let runControllerEntryPoint: string | undefined; + let indexWorkerEntryPoint: string | undefined; + let indexControllerEntryPoint: string | undefined; for (const [outputPath, outputMeta] of Object.entries(result.metafile.outputs)) { if (outputPath.endsWith(".mjs")) { @@ -161,12 +164,14 @@ export async function getBundleResultFromBuild( configPath = $outputPath; } else if (isLoaderEntryPoint(outputMeta.entryPoint)) { loaderEntryPoint = $outputPath; - } else if (isWorkerEntryPointForTarget(outputMeta.entryPoint, target)) { - workerEntryPoint = $outputPath; - } else if (isIndexerEntryPointForTarget(outputMeta.entryPoint, target)) { - indexerEntryPoint = $outputPath; - } else if (isExecutorEntryPointForTarget(outputMeta.entryPoint, target)) { - executorEntryPoint = $outputPath; + } else if (isRunControllerForTarget(outputMeta.entryPoint, target)) { + runControllerEntryPoint = $outputPath; + } else if (isRunWorkerForTarget(outputMeta.entryPoint, target)) { + runWorkerEntryPoint = $outputPath; + } else if (isIndexControllerForTarget(outputMeta.entryPoint, target)) { + indexControllerEntryPoint = $outputPath; + } else if (isIndexWorkerForTarget(outputMeta.entryPoint, target)) { + indexWorkerEntryPoint = $outputPath; } else { if ( !outputMeta.entryPoint.startsWith("..") && @@ -189,9 +194,10 @@ export async function getBundleResultFromBuild( files, configPath: configPath, loaderEntryPoint, - workerEntryPoint, - executorEntryPoint, - indexerEntryPoint, + runWorkerEntryPoint, + runControllerEntryPoint, + indexWorkerEntryPoint, + indexControllerEntryPoint, contentHash: hasher.digest("hex"), }; } diff --git a/packages/cli-v3/src/build/manifests.ts b/packages/cli-v3/src/build/manifests.ts index 1a9fd620e..04fe2e3f6 100644 --- a/packages/cli-v3/src/build/manifests.ts +++ b/packages/cli-v3/src/build/manifests.ts @@ -17,7 +17,14 @@ export async function copyManifestToDir( updatedManifest.configPath = updatedManifest.configPath.replace(source, destination); updatedManifest.loaderEntryPoint = updatedManifest.loaderEntryPoint?.replace(source, destination); - updatedManifest.workerEntryPoint = updatedManifest.workerEntryPoint?.replace(source, destination); + updatedManifest.runWorkerEntryPoint = updatedManifest.runWorkerEntryPoint.replace( + source, + destination + ); + updatedManifest.indexWorkerEntryPoint = updatedManifest.indexWorkerEntryPoint.replace( + source, + destination + ); updatedManifest.files = updatedManifest.files.map((file) => { return { diff --git a/packages/cli-v3/src/build/packageModules.ts b/packages/cli-v3/src/build/packageModules.ts index b1574d2a4..423e67cc1 100644 --- a/packages/cli-v3/src/build/packageModules.ts +++ b/packages/cli-v3/src/build/packageModules.ts @@ -2,20 +2,22 @@ import { join } from "node:path"; import { sourceDir } from "../sourceDir.js"; import { BuildTarget } from "@trigger.dev/core/v3"; -export const devExecutorEntryPoint = join(sourceDir, "entryPoints", "dev-executor.js"); -export const devIndexerEntryPoint = join(sourceDir, "entryPoints", "dev-indexer.js"); +export const devRunWorker = join(sourceDir, "entryPoints", "dev-run-worker.js"); +export const devIndexWorker = join(sourceDir, "entryPoints", "dev-index-worker.js"); -export const deployEntryPoint = join(sourceDir, "entryPoints", "deploy.js"); -export const deployExecutorEntryPoint = join(sourceDir, "entryPoints", "deploy-executor.js"); -export const deployIndexerEntryPoint = join(sourceDir, "entryPoints", "deploy-indexer.js"); +export const deployRunController = join(sourceDir, "entryPoints", "deploy-run-controller.js"); +export const deployRunWorker = join(sourceDir, "entryPoints", "deploy-run-worker.js"); +export const deployIndexController = join(sourceDir, "entryPoints", "deploy-index-controller.js"); +export const deployIndexWorker = join(sourceDir, "entryPoints", "deploy-index-worker.js"); export const telemetryEntryPoint = join(sourceDir, "entryPoints", "loader.js"); -export const devEntryPoints = [devExecutorEntryPoint, devIndexerEntryPoint, telemetryEntryPoint]; +export const devEntryPoints = [devRunWorker, devIndexWorker, telemetryEntryPoint]; export const deployEntryPoints = [ - deployIndexerEntryPoint, - deployExecutorEntryPoint, - deployEntryPoint, + deployRunController, + deployRunWorker, + deployIndexController, + deployIndexWorker, telemetryEntryPoint, ]; @@ -23,41 +25,61 @@ export const esmShimPath = join(sourceDir, "shims", "esm.js"); export const shims = [esmShimPath]; -function isDevExecutorEntryPoint(entryPoint: string) { - return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-executor.js")); +function isDevRunWorker(entryPoint: string) { + return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-run-worker.js")); } -function isDevIndexerEntryPoint(entryPoint: string) { - return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-indexer.js")); +function isDevIndexWorker(entryPoint: string) { + return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-index-worker.js")); } -function isDeployIndexerEntryPoint(entryPoint: string) { - return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-indexer.js")); +function isDeployIndexController(entryPoint: string) { + return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-index-controller.js")); } -function isDeployEntryPoint(entryPoint: string) { - return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy.js")); +function isDeployIndexWorker(entryPoint: string) { + return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-index-worker.js")); } -function isDeployExecutorEntryPoint(entryPoint: string) { - return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-executor.js")); +function isDeployRunController(entryPoint: string) { + return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-run-controller.js")); +} + +function isDeployRunWorker(entryPoint: string) { + return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-run-worker.js")); } export function isLoaderEntryPoint(entryPoint: string) { return entryPoint.includes(join("dist", "esm", "entryPoints", "loader.js")); } -export function isExecutorEntryPointForTarget(entryPoint: string, target: BuildTarget) { +export function isRunWorkerForTarget(entryPoint: string, target: BuildTarget) { if (target === "dev") { - return isDevExecutorEntryPoint(entryPoint); + return isDevRunWorker(entryPoint); } else { - return isDeployExecutorEntryPoint(entryPoint); + return isDeployRunWorker(entryPoint); } } -export function isWorkerEntryPointForTarget(entryPoint: string, target: BuildTarget) { +export function isRunControllerForTarget(entryPoint: string, target: BuildTarget) { if (target === "deploy") { - return isDeployEntryPoint(entryPoint); + return isDeployRunController(entryPoint); + } + + return false; +} + +export function isIndexWorkerForTarget(entryPoint: string, target: BuildTarget) { + if (target === "dev") { + return isDevIndexWorker(entryPoint); + } else { + return isDeployIndexWorker(entryPoint); + } +} + +export function isIndexControllerForTarget(entryPoint: string, target: BuildTarget) { + if (target === "deploy") { + return isDeployIndexController(entryPoint); } return false; @@ -66,11 +88,3 @@ export function isWorkerEntryPointForTarget(entryPoint: string, target: BuildTar export function isConfigEntryPoint(entryPoint: string) { return entryPoint.startsWith("trigger.config.ts"); } - -export function isIndexerEntryPointForTarget(entryPoint: string, target: BuildTarget) { - if (target === "dev") { - return isDevIndexerEntryPoint(entryPoint); - } else { - return isDeployIndexerEntryPoint(entryPoint); - } -} diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index ab5604ac6..9c1c2a7d9 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -1,5 +1,5 @@ import { intro, outro } from "@clack/prompts"; -import { CORE_VERSION } from "@trigger.dev/core/v3"; +import { CORE_VERSION, prepareDeploymentError } from "@trigger.dev/core/v3"; import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build"; import { BuildManifest, InitializeDeploymentResponseBody } from "@trigger.dev/core/v3/schemas"; import { Command, Option as CommandOption } from "commander"; @@ -16,10 +16,12 @@ import { resolvePluginsForContext, } from "../build/extensions.js"; import { createExternalsBuildExtension } from "../build/externals.js"; +import { getInstrumentedPackageNames } from "../build/instrumentation.js"; import { - deployEntryPoint, - deployExecutorEntryPoint, - deployIndexerEntryPoint, + deployIndexController, + deployIndexWorker, + deployRunController, + deployRunWorker, telemetryEntryPoint, } from "../build/packageModules.js"; import { @@ -39,7 +41,8 @@ import { saveLogs, } from "../deploy/logs.js"; import { buildManifestToJSON } from "../utilities/buildManifest.js"; -import { chalkError, cliLink } from "../utilities/cliOutput.js"; +import { chalkError, cliLink, prettyError } from "../utilities/cliOutput.js"; +import { loadDotEnvVars } from "../utilities/dotEnv.js"; import { writeJSONFile } from "../utilities/fileSystem.js"; import { printStandloneInitialBanner } from "../utilities/initialBanner.js"; import { logger } from "../utilities/logger.js"; @@ -50,8 +53,6 @@ import { spinner } from "../utilities/windows.js"; import { VERSION } from "../version.js"; import { login } from "./login.js"; import { updateTriggerPackages } from "./update.js"; -import { loadDotEnvVars, resolveDotEnvVars } from "../utilities/dotEnv.js"; -import { getInstrumentedPackageNames } from "../build/instrumentation.js"; const DeployCommandOptions = CommonCommandOptions.extend({ dryRun: z.boolean().default(false), @@ -250,9 +251,10 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { dirs: resolvedConfig.dirs, }, outputPath: destination.path, - workerEntryPoint: bundleResult.workerEntryPoint ?? deployEntryPoint, - executorEntryPoint: bundleResult.executorEntryPoint ?? deployExecutorEntryPoint, - indexerEntryPoint: bundleResult.indexerEntryPoint ?? deployIndexerEntryPoint, + runControllerEntryPoint: bundleResult.runControllerEntryPoint ?? deployRunController, + runWorkerEntryPoint: bundleResult.runWorkerEntryPoint ?? deployRunWorker, + indexControllerEntryPoint: bundleResult.indexControllerEntryPoint ?? deployIndexController, + indexWorkerEntryPoint: bundleResult.indexWorkerEntryPoint ?? deployIndexWorker, loaderEntryPoint: bundleResult.loaderEntryPoint ?? telemetryEntryPoint, configPath: bundleResult.configPath, deploy: { @@ -484,11 +486,14 @@ function rewriteBuildManifestPaths( })), outputPath: rewriteOutputPath(destinationDir, buildManifest.outputPath), configPath: rewriteOutputPath(destinationDir, buildManifest.configPath), - executorEntryPoint: rewriteOutputPath(destinationDir, buildManifest.executorEntryPoint), - indexerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.indexerEntryPoint), - workerEntryPoint: buildManifest.workerEntryPoint - ? rewriteOutputPath(destinationDir, buildManifest.workerEntryPoint) + runControllerEntryPoint: buildManifest.runControllerEntryPoint + ? rewriteOutputPath(destinationDir, buildManifest.runControllerEntryPoint) : undefined, + runWorkerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.runWorkerEntryPoint), + indexControllerEntryPoint: buildManifest.indexControllerEntryPoint + ? rewriteOutputPath(destinationDir, buildManifest.indexControllerEntryPoint) + : undefined, + indexWorkerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.indexWorkerEntryPoint), loaderEntryPoint: buildManifest.loaderEntryPoint ? rewriteOutputPath(destinationDir, buildManifest.loaderEntryPoint) : undefined, @@ -550,15 +555,15 @@ function rewriteOutputPath(destinationDir: string, filePath: string) { } async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) { - if (!buildManifest.workerEntryPoint) { - throw new Error("No worker entry point found in build manifest"); + if (!buildManifest.runControllerEntryPoint || !buildManifest.indexControllerEntryPoint) { + throw new Error("Something went wrong with the build. Aborting deployment. [code 7789]"); } const containerfile = await generateContainerfile({ runtime: buildManifest.runtime, - workerEntryPoint: buildManifest.workerEntryPoint, + entrypoint: buildManifest.runControllerEntryPoint, build: buildManifest.build, - indexerEntryPoint: buildManifest.indexerEntryPoint, + indexScript: buildManifest.indexControllerEntryPoint, }); await writeFile(join(outputPath, "Containerfile"), containerfile); @@ -589,25 +594,94 @@ async function failDeploy( ) { $spinner.stop(`Failed to deploy project`); - // If there are logs, let's write it out to a temporary file and include the path in the error message - if (logs.trim() !== "") { - const logPath = await saveLogs(deployment.shortCode, logs); + const doOutputLogs = async (prefix: string = "Error") => { + if (logs.trim() !== "") { + const logPath = await saveLogs(deployment.shortCode, logs); - printWarnings(warnings); - printErrors(errors); + printWarnings(warnings); + printErrors(errors); - checkLogsForErrors(logs); + checkLogsForErrors(logs); - outro( - `${chalkError("Error:")} ${error.message}. Full build logs have been saved to ${logPath}` - ); + outro( + `${chalkError(`${prefix}:`)} ${ + error.message + }. Full build logs have been saved to ${logPath}` + ); + } else { + outro(`${chalkError(`${prefix}:`)} ${error.message}.`); + } + }; + + const exitCommand = (message: string) => { + throw new SkipLoggingError(message); + }; + + const deploymentResponse = await client.getDeployment(deployment.id); + + if (!deploymentResponse.success) { + logger.debug(`Failed to get deployment with worker: ${deploymentResponse.error}`); } else { - outro(`${chalkError("Error:")} ${error.message}.`); + const serverDeployment = deploymentResponse.data; + + switch (serverDeployment.status) { + case "PENDING": + case "DEPLOYING": + case "BUILDING": { + await doOutputLogs(); + + await client.failDeployment(deployment.id, { + error, + }); + + exitCommand("Failed to deploy project"); + + break; + } + case "CANCELED": { + await doOutputLogs("Canceled"); + + exitCommand("Failed to deploy project"); + + break; + } + case "FAILED": { + const errorData = serverDeployment.errorData + ? prepareDeploymentError(serverDeployment.errorData) + : undefined; + + if (errorData) { + prettyError(errorData.name, errorData.stack, errorData.stderr); + + if (logs.trim() !== "") { + const logPath = await saveLogs(deployment.shortCode, logs); + + outro(`Aborting deployment. Full build logs have been saved to ${logPath}`); + } else { + outro(`Aborting deployment`); + } + } else { + await doOutputLogs("Failed"); + } + + exitCommand("Failed to deploy project"); + + break; + } + case "DEPLOYED": { + await doOutputLogs("Deployed with errors"); + + exitCommand("Deployed with errors"); + + break; + } + case "TIMED_OUT": { + await doOutputLogs("TimedOut"); + + exitCommand("Timed out"); + + break; + } + } } - - await client.failDeployment(deployment.id, { - error, - }); - - throw new SkipLoggingError(`Failed to deploy: ${error.message}`); } diff --git a/packages/cli-v3/src/commands/dev.tsx b/packages/cli-v3/src/commands/dev.tsx index 084644f98..ddf88de32 100644 --- a/packages/cli-v3/src/commands/dev.tsx +++ b/packages/cli-v3/src/commands/dev.tsx @@ -12,6 +12,7 @@ import { logger } from "../utilities/logger.js"; import { runtimeChecks } from "../utilities/runtimeCheck.js"; import { getProjectClient, isLoggedIn, LoginResultOk } from "../utilities/session.js"; import { updateTriggerPackages } from "./update.js"; +import { login } from "./login.js"; const DevCommandOptions = CommonCommandOptions.extend({ debugOtel: z.boolean().default(false), @@ -45,7 +46,11 @@ export function configureDevCommand(program: Command) { export async function devCommand(options: DevCommandOptions) { runtimeChecks(); - const authorization = await isLoggedIn(options.profile); + const authorization = await login({ + embedded: true, + defaultApiUrl: options.apiUrl, + profile: options.profile, + }); if (!authorization.ok) { if (authorization.error === "fetch failed") { diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index fd843efbb..f4aa318f7 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -22,6 +22,7 @@ import { logger } from "../utilities/logger.js"; import { spinner } from "../utilities/windows.js"; import { isLinuxServer } from "../utilities/linux.js"; import { VERSION } from "../version.js"; +import { env } from "std-env"; export const LoginCommandOptions = CommonCommandOptions.extend({ apiUrl: z.string(), @@ -74,12 +75,12 @@ export async function login(options?: LoginOptions): Promise { intro("Logging in to Trigger.dev"); } - const accessTokenFromEnv = process.env.TRIGGER_ACCESS_TOKEN; + const accessTokenFromEnv = env.TRIGGER_ACCESS_TOKEN; if (accessTokenFromEnv) { const auth = { accessToken: accessTokenFromEnv, - apiUrl: process.env.TRIGGER_API_URL ?? opts.defaultApiUrl ?? "https://api.trigger.dev", + apiUrl: env.TRIGGER_API_URL ?? opts.defaultApiUrl ?? "https://api.trigger.dev", }; const apiClient = new CliApiClient(auth.apiUrl, auth.accessToken); const userData = await apiClient.whoAmI(); diff --git a/packages/cli-v3/src/commands/update.ts b/packages/cli-v3/src/commands/update.ts index 5d29b57fc..0197e4288 100644 --- a/packages/cli-v3/src/commands/update.ts +++ b/packages/cli-v3/src/commands/update.ts @@ -11,6 +11,7 @@ import { printStandloneInitialBanner, updateCheck } from "../utilities/initialBa import { logger } from "../utilities/logger.js"; import { spinner } from "../utilities/windows.js"; import { VERSION } from "../version.js"; +import { hasTTY } from "std-env"; export const UpdateCommandOptions = CommonCommandOptions.pick({ logLevel: true, @@ -142,7 +143,7 @@ export async function updateTriggerPackages( ); } - if (!process.stdout.isTTY) { + if (!hasTTY) { // Running in CI with version mismatch detected outro("Deploy failed"); diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 34ab405d6..414aabaca 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -408,8 +408,8 @@ function extractImageDigest(outputs: string[]) { export type GenerateContainerfileOptions = { runtime: BuildRuntime; build: BuildManifest["build"]; - indexerEntryPoint: string; - workerEntryPoint: string; + indexScript: string; + entrypoint: string; }; export async function generateContainerfile(options: GenerateContainerfileOptions) { @@ -481,7 +481,7 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \ NODE_ENV=production # Run the indexer -RUN bun run ${options.indexerEntryPoint} +RUN bun run ${options.indexScript} # Development or production stage builds upon the base stage FROM base AS final @@ -510,7 +510,7 @@ COPY --from=install --chown=bun:bun /app ./ # Copy the index.json file from the indexer stage COPY --from=indexer --chown=bun:bun /app/index.json ./ -ENTRYPOINT [ "dumb-init", "node", "${options.workerEntryPoint}" ] +ENTRYPOINT [ "dumb-init", "node", "${options.entrypoint}" ] CMD [] `; } @@ -579,7 +579,7 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \ NODE_OPTIONS="--max_old_space_size=8192" # Run the indexer -RUN node ${options.indexerEntryPoint} +RUN node ${options.indexScript} # Development or production stage builds upon the base stage FROM base AS final @@ -609,7 +609,7 @@ COPY --from=install --chown=node:node /app ./ # Copy the index.json file from the indexer stage COPY --from=indexer --chown=node:node /app/index.json ./ -ENTRYPOINT [ "dumb-init", "node", "${options.workerEntryPoint}" ] +ENTRYPOINT [ "dumb-init", "node", "${options.entrypoint}" ] CMD [] `; } diff --git a/packages/cli-v3/src/dev/backgroundWorker.ts b/packages/cli-v3/src/dev/backgroundWorker.ts index 283129951..f8afcc3ea 100644 --- a/packages/cli-v3/src/dev/backgroundWorker.ts +++ b/packages/cli-v3/src/dev/backgroundWorker.ts @@ -10,27 +10,23 @@ import { TaskRunFailedExecutionResult, WorkerManifest, correctErrorStackTrace, - indexerToWorkerMessages, } from "@trigger.dev/core/v3"; -import { parseMessageFromCatalog } from "@trigger.dev/core/v3/zodMessageHandler"; import { Evt } from "evt"; -import { fork } from "node:child_process"; -import { execPathForRuntime } from "@trigger.dev/core/v3/build"; import { join } from "node:path"; -import { TaskRunProcess, TaskRunProcessOptions } from "../executions/taskRunProcess.js"; -import { eventBus } from "../utilities/eventBus.js"; -import { writeJSONFile } from "../utilities/fileSystem.js"; -import { logger } from "../utilities/logger.js"; import { CancelledProcessError, CleanupProcessError, SigKillTimeoutProcessError, - TaskMetadataParseError, - UncaughtExceptionError, UnexpectedExitError, getFriendlyErrorMessage, -} from "../executions/errors.js"; +} from "@trigger.dev/core/v3/errors"; +import { TaskRunProcess, TaskRunProcessOptions } from "../executions/taskRunProcess.js"; +import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js"; +import { prettyError } from "../utilities/cliOutput.js"; +import { eventBus } from "../utilities/eventBus.js"; +import { writeJSONFile } from "../utilities/fileSystem.js"; +import { logger } from "../utilities/logger.js"; export type CurrentWorkers = BackgroundWorkerCoordinator["currentWorkers"]; export class BackgroundWorkerCoordinator { @@ -254,87 +250,36 @@ export class BackgroundWorker { throw new Error("Worker already initialized"); } - let resolved = false; - // Write the build manifest to this.build.outputPath/build.json await writeJSONFile(this.buildManifestPath, this.build, true); - logger.debug("Initializing worker", { build: this.build, params: this.params }); + logger.debug("indexing worker manifest", { build: this.build, params: this.params }); - this.manifest = await new Promise((resolve, reject) => { - const child = fork(this.build.indexerEntryPoint, { - stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"], - cwd: this.params.cwd, - env: { - ...this.params.env, - TRIGGER_BUILD_MANIFEST_PATH: this.buildManifestPath, - NODE_OPTIONS: this.build.loaderEntryPoint - ? `--import=${this.build.loaderEntryPoint} ${process.env.NODE_OPTIONS ?? ""}` - : process.env.NODE_OPTIONS, - }, - execPath: execPathForRuntime(this.build.runtime), - }); - - // Set a timeout to kill the child process if it doesn't respond - const timeout = setTimeout(() => { - if (resolved) { - return; + this.manifest = await indexWorkerManifest({ + runtime: this.build.runtime, + indexWorkerPath: this.build.indexWorkerEntryPoint, + buildManifestPath: this.buildManifestPath, + nodeOptions: this.build.loaderEntryPoint + ? `--import=${this.build.loaderEntryPoint}` + : undefined, + env: this.params.env, + cwd: this.params.cwd, + otelHookInclude: this.build.otelImportHook?.include, + otelHookExclude: this.build.otelImportHook?.exclude, + handleStdout(data) { + logger.debug(data); + }, + handleStderr(data) { + if (!data.includes("Debugger attached")) { + prettyError(data.toString()); } - - resolved = true; - child.kill(); - reject(new Error("Worker timed out")); - }, 20_000); - - child.on("message", async (msg: any) => { - const message = parseMessageFromCatalog(msg, indexerToWorkerMessages); - - switch (message.type) { - case "INDEX_COMPLETE": { - clearTimeout(timeout); - resolved = true; - resolve(message.payload.manifest); - child.kill(); - break; - } - case "TASKS_FAILED_TO_PARSE": { - clearTimeout(timeout); - resolved = true; - reject(new TaskMetadataParseError(message.payload.zodIssues, message.payload.tasks)); - child.kill(); - break; - } - case "UNCAUGHT_EXCEPTION": { - clearTimeout(timeout); - resolved = true; - reject(new UncaughtExceptionError(message.payload.error, message.payload.origin)); - child.kill(); - break; - } - } - }); - - child.on("exit", (code) => { - if (!resolved) { - clearTimeout(timeout); - resolved = true; - reject(new Error(`Worker exited with code ${code}`)); - } - }); - - child.stdout?.on("data", (data) => { - logger.debug(`indexer: ${data.toString()}`); - }); - - child.stderr?.on("data", (data) => { - logger.debug(`indexer: ${data.toString()}`); - }); + }, }); // Write the build manifest to this.build.outputPath/worker.json await writeJSONFile(this.workerManifestPath, this.manifest, true); - logger.debug("Worker initialized", { path: this.build.outputPath }); + logger.debug("worker manifest indexed", { path: this.build.outputPath }); } // We need to notify all the task run processes that a task run has completed, diff --git a/packages/cli-v3/src/dev/devOutput.ts b/packages/cli-v3/src/dev/devOutput.ts index 5b2ab9f1d..ae2ad5792 100644 --- a/packages/cli-v3/src/dev/devOutput.ts +++ b/packages/cli-v3/src/dev/devOutput.ts @@ -11,11 +11,21 @@ import { chalkWarning, chalkWorker, cliLink, + prettyError, prettyPrintDate, } from "../utilities/cliOutput.js"; import { eventBus, EventBusEventArgs } from "../utilities/eventBus.js"; -import { TaskRunError, TaskRunErrorCodes } from "@trigger.dev/core/v3/schemas"; +import { + TaskMetadataFailedToParseData, + TaskRunError, + TaskRunErrorCodes, +} from "@trigger.dev/core/v3/schemas"; import { formatDurationMilliseconds } from "@trigger.dev/core/v3"; +import { + createTaskMetadataFailedErrorStack, + TaskIndexingImportError, + TaskMetadataParseError, +} from "@trigger.dev/core/v3/errors"; export type DevOutputOptions = { name: string | undefined; @@ -63,6 +73,29 @@ export function startDevOutput(options: DevOutputOptions) { ); }; + const backgroundWorkerIndexingError = ( + ...[buildManifest, error]: EventBusEventArgs<"backgroundWorkerIndexingError"> + ) => { + if (error instanceof TaskIndexingImportError) { + for (const importError of error.importErrors) { + prettyError(`Could not import ${importError.file}`, importError.stack); + } + } else if (error instanceof TaskMetadataParseError) { + const errorStack = createTaskMetadataFailedErrorStack({ + version: "v1", + zodIssues: error.zodIssues, + tasks: error.tasks, + }); + + prettyError(`Could not parse task metadata`, errorStack); + } else { + const errorText = error instanceof Error ? error.message : "Unknown error"; + const stack = error instanceof Error ? error.stack : undefined; + + prettyError(`Build failed: ${errorText}`, stack); + } + }; + const runStarted = (...[worker, payload]: EventBusEventArgs<"runStarted">) => { if (!worker.serverWorker) { return; @@ -142,6 +175,7 @@ export function startDevOutput(options: DevOutputOptions) { eventBus.on("backgroundWorkerInitialized", backgroundWorkerInitialized); eventBus.on("runStarted", runStarted); eventBus.on("runCompleted", runCompleted); + eventBus.on("backgroundWorkerIndexingError", backgroundWorkerIndexingError); return () => { eventBus.off("rebuildStarted", rebuildStarted); @@ -150,6 +184,7 @@ export function startDevOutput(options: DevOutputOptions) { eventBus.off("backgroundWorkerInitialized", backgroundWorkerInitialized); eventBus.off("runStarted", runStarted); eventBus.off("runCompleted", runCompleted); + eventBus.off("backgroundWorkerIndexingError", backgroundWorkerIndexingError); }; } diff --git a/packages/cli-v3/src/dev/devSession.ts b/packages/cli-v3/src/dev/devSession.ts index 41d70e4ca..4842c33bb 100644 --- a/packages/cli-v3/src/dev/devSession.ts +++ b/packages/cli-v3/src/dev/devSession.ts @@ -18,11 +18,7 @@ import { } from "../build/extensions.js"; import { createExternalsBuildExtension } from "../build/externals.js"; import { copyManifestToDir } from "../build/manifests.js"; -import { - devExecutorEntryPoint, - devIndexerEntryPoint, - telemetryEntryPoint, -} from "../build/packageModules.js"; +import { devIndexWorker, devRunWorker, telemetryEntryPoint } from "../build/packageModules.js"; import { type DevCommandOptions } from "../commands/dev.js"; import { eventBus } from "../utilities/eventBus.js"; import { logger } from "../utilities/logger.js"; @@ -31,6 +27,7 @@ import { EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js"; import { VERSION } from "../version.js"; import { startDevOutput } from "./devOutput.js"; import { startWorkerRuntime } from "./workerRuntime.js"; +import { getInstrumentedPackageNames } from "../build/instrumentation.js"; export type DevSessionOptions = { name: string | undefined; @@ -75,21 +72,25 @@ export async function startDevSession({ const pluginsFromExtensions = resolvePluginsForContext(buildContext); async function updateBundle(bundle: BundleResult, workerDir?: EphemeralDirectory) { + let buildManifest = await createBuildManifestFromBundle( + bundle, + destination.path, + rawConfig, + workerDir?.path + ); + + buildManifest = await notifyExtensionOnBuildComplete(buildContext, buildManifest); + try { - let buildManifest = await createBuildManifestFromBundle( - bundle, - destination.path, - rawConfig, - workerDir?.path - ); - - buildManifest = await notifyExtensionOnBuildComplete(buildContext, buildManifest); - logger.debug("Updated bundle", { bundle, buildManifest }); await runtime.initializeWorker(buildManifest); } catch (error) { - logger.error("Error updating bundle", { error }); + if (error instanceof Error) { + eventBus.emit("backgroundWorkerIndexingError", buildManifest, error); + } else { + logger.error("Error updating bundle", { error }); + } } } @@ -191,14 +192,17 @@ async function createBuildManifestFromBundle( dirs: resolvedConfig.dirs, }, outputPath: destination, - executorEntryPoint: bundle.executorEntryPoint ?? devExecutorEntryPoint, + runWorkerEntryPoint: bundle.runWorkerEntryPoint ?? devRunWorker, + indexWorkerEntryPoint: bundle.indexWorkerEntryPoint ?? devIndexWorker, loaderEntryPoint: bundle.loaderEntryPoint ?? telemetryEntryPoint, - indexerEntryPoint: bundle.indexerEntryPoint ?? devIndexerEntryPoint, configPath: bundle.configPath, deploy: { env: {}, }, build: {}, + otelImportHook: { + include: getInstrumentedPackageNames(resolvedConfig), + }, }; if (!workerDir) { diff --git a/packages/cli-v3/src/dev/workerRuntime.ts b/packages/cli-v3/src/dev/workerRuntime.ts index 338220e85..c51aae0d5 100644 --- a/packages/cli-v3/src/dev/workerRuntime.ts +++ b/packages/cli-v3/src/dev/workerRuntime.ts @@ -26,6 +26,7 @@ import { eventBus } from "../utilities/eventBus.js"; import { logger } from "../utilities/logger.js"; import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js"; import { BackgroundWorker, BackgroundWorkerCoordinator } from "./backgroundWorker.js"; +import { env } from "std-env"; export interface WorkerRuntime { shutdown(): Promise; @@ -316,13 +317,13 @@ function WebsocketFactory(apiKey: string) { } function gatherProcessEnv() { - const env = { - ...process.env, + const $env = { + ...env, NODE_ENV: "development", }; // Filter out undefined values - return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined)); + return Object.fromEntries(Object.entries($env).filter(([key, value]) => value !== undefined)); } function validateWorkerManifest(manifest: WorkerManifest): string[] { diff --git a/packages/cli-v3/src/entryPoints/deploy-index-controller.ts b/packages/cli-v3/src/entryPoints/deploy-index-controller.ts new file mode 100644 index 000000000..47c8dd417 --- /dev/null +++ b/packages/cli-v3/src/entryPoints/deploy-index-controller.ts @@ -0,0 +1,116 @@ +import { + BuildManifest, + CreateBackgroundWorkerRequestBody, + serializeIndexingError, +} from "@trigger.dev/core/v3"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { env } from "std-env"; +import { CliApiClient } from "../apiClient.js"; +import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js"; +import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js"; + +async function loadBuildManifest() { + const manifestContents = await readFile("./build.json", "utf-8"); + const raw = JSON.parse(manifestContents); + + return BuildManifest.parse(raw); +} + +async function bootstrap() { + const buildManifest = await loadBuildManifest(); + + if (typeof env.TRIGGER_API_URL !== "string") { + console.error("TRIGGER_API_URL is not set"); + process.exit(1); + } + + const cliApiClient = new CliApiClient(env.TRIGGER_API_URL, env.TRIGGER_SECRET_KEY); + + if (!env.TRIGGER_PROJECT_REF) { + console.error("TRIGGER_PROJECT_REF is not set"); + process.exit(1); + } + + if (!env.TRIGGER_DEPLOYMENT_ID) { + console.error("TRIGGER_DEPLOYMENT_ID is not set"); + process.exit(1); + } + + return { + buildManifest, + cliApiClient, + projectRef: env.TRIGGER_PROJECT_REF, + deploymentId: env.TRIGGER_DEPLOYMENT_ID, + }; +} + +type BootstrapResult = Awaited>; + +async function indexDeployment({ + cliApiClient, + projectRef, + deploymentId, + buildManifest, +}: BootstrapResult) { + const stdout: string[] = []; + const stderr: string[] = []; + + try { + const $env = await cliApiClient.getEnvironmentVariables(projectRef); + + if (!$env.success) { + throw new Error(`Failed to fetch environment variables: ${env.error}`); + } + + const workerManifest = await indexWorkerManifest({ + runtime: buildManifest.runtime, + indexWorkerPath: buildManifest.indexWorkerEntryPoint, + buildManifestPath: "./build.json", + nodeOptions: buildManifest.loaderEntryPoint + ? `--import=${buildManifest.loaderEntryPoint}` + : undefined, + env: $env.data.variables, + otelHookExclude: buildManifest.otelImportHook?.exclude, + otelHookInclude: buildManifest.otelImportHook?.include, + handleStdout(data) { + stdout.push(data); + }, + handleStderr(data) { + if (!data.includes("DeprecationWarning")) { + stderr.push(data); + } + }, + }); + + console.log("Writing index.json", process.cwd()); + + await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2)); + + const sourceFiles = resolveTaskSourceFiles(buildManifest.sources, workerManifest.tasks); + + const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = { + localOnly: true, + metadata: { + contentHash: buildManifest.contentHash, + packageVersion: buildManifest.packageVersion, + cliPackageVersion: buildManifest.cliPackageVersion, + tasks: workerManifest.tasks, + sourceFiles, + }, + supportsLazyAttempts: true, + }; + + await cliApiClient.createDeploymentBackgroundWorker(deploymentId, backgroundWorkerBody); + } catch (error) { + const serialiedIndexError = serializeIndexingError(error, stderr.join("\n")); + + await cliApiClient.failDeployment(deploymentId, { error: serialiedIndexError }); + + process.exit(1); + } +} + +const results = await bootstrap(); + +await indexDeployment(results); diff --git a/packages/cli-v3/src/entryPoints/dev-indexer.ts b/packages/cli-v3/src/entryPoints/deploy-index-worker.ts similarity index 69% rename from packages/cli-v3/src/entryPoints/dev-indexer.ts rename to packages/cli-v3/src/entryPoints/deploy-index-worker.ts index 2256aca44..55500eb66 100644 --- a/packages/cli-v3/src/entryPoints/dev-indexer.ts +++ b/packages/cli-v3/src/entryPoints/deploy-index-worker.ts @@ -13,6 +13,8 @@ import { import { sendMessageInCatalog, ZodSchemaParsedError } from "@trigger.dev/core/v3/zodMessageHandler"; import { readFile } from "node:fs/promises"; import sourceMapSupport from "source-map-support"; +import { registerTasks } from "../indexing/registerTasks.js"; +import { env } from "std-env"; sourceMapSupport.install({ handleUncaughtExceptions: false, @@ -63,7 +65,7 @@ async function importConfig( } async function loadBuildManifest() { - const manifestContents = await readFile(process.env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8"); + const manifestContents = await readFile(env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8"); const raw = JSON.parse(manifestContents); return BuildManifest.parse(raw); @@ -76,42 +78,23 @@ async function bootstrap() { // This needs to run or the PrismaInstrumentation will throw an error const tracingSDK = new TracingSDK({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", instrumentations: config.instrumentations ?? [], - diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", forceFlushTimeoutMillis: 30_000, }); - for (const file of buildManifest.files) { - const module = await import(file.out); - - for (const exportName of getExportNames(module)) { - const task = module[exportName] ?? module.default?.[exportName]; - - if (!task) { - continue; - } - - if (task[Symbol.for("trigger.dev/task")]) { - if (taskCatalog.taskExists(task.id)) { - taskCatalog.registerTaskFileMetadata(task.id, { - exportName, - filePath: file.entry, - entryPoint: file.out, - }); - } - } - } - } + const importErrors = await registerTasks(buildManifest); return { tracingSDK, config, buildManifest, + importErrors, }; } -const { buildManifest } = await bootstrap(); +const { buildManifest, importErrors } = await bootstrap(); const tasks = taskCatalog.listTaskManifests(); @@ -123,10 +106,11 @@ await sendMessageInCatalog( tasks, configPath: buildManifest.configPath, runtime: buildManifest.runtime, - executorEntryPoint: buildManifest.executorEntryPoint, - workerEntryPoint: buildManifest.workerEntryPoint, + workerEntryPoint: buildManifest.runWorkerEntryPoint, + controllerEntryPoint: buildManifest.runControllerEntryPoint, loaderEntryPoint: buildManifest.loaderEntryPoint, }, + importErrors, }, async (msg) => { process.send?.(msg); @@ -153,19 +137,3 @@ await new Promise((resolve) => { resolve(); }, 10); }); - -function getExportNames(module: any) { - const exports: string[] = []; - - const exportKeys = Object.keys(module); - - if (exportKeys.length === 0) { - return exports; - } - - if (exportKeys.length === 1 && exportKeys[0] === "default") { - return Object.keys(module.default); - } - - return exportKeys; -} diff --git a/packages/cli-v3/src/entryPoints/deploy-indexer.ts b/packages/cli-v3/src/entryPoints/deploy-indexer.ts deleted file mode 100644 index 3de0276b3..000000000 --- a/packages/cli-v3/src/entryPoints/deploy-indexer.ts +++ /dev/null @@ -1,261 +0,0 @@ -import { - BuildManifest, - CreateBackgroundWorkerRequestBody, - type HandleErrorFunction, - taskCatalog, - TriggerConfig, - WorkerManifest, -} from "@trigger.dev/core/v3"; -import { - StandardTaskCatalog, - TracingDiagnosticLogLevel, - TracingSDK, -} from "@trigger.dev/core/v3/workers"; -import { readFile, writeFile } from "node:fs/promises"; -import sourceMapSupport from "source-map-support"; -import { CliApiClient } from "../apiClient.js"; -import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js"; -import { join } from "node:path"; - -sourceMapSupport.install({ - handleUncaughtExceptions: false, - environment: "node", - hookRequire: false, -}); - -taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog()); - -async function importConfig(configPath: string): Promise<{ - config: TriggerConfig; - handleError?: HandleErrorFunction; -}> { - const configModule = await import(configPath); - - const config = configModule?.default ?? configModule?.config; - - return { - config, - handleError: configModule?.handleError, - }; -} - -async function loadBuildManifest() { - const manifestContents = await readFile("./build.json", "utf-8"); - const raw = JSON.parse(manifestContents); - - return BuildManifest.parse(raw); -} - -// We need to make sure, that if any errors are thrown, that we fail the deployment - -// 1. Fetch the build manifest -// 2. Fetch the environment variables from the server -// 3. Import the config -// 5. Inject the env vars into process.env -// 6. Configure the tracing SDK -// 7. Load all the tasks from the build manifest and create the index.json -// 8. Write the index.json to the file system -// 9. Update the deployment with the worker index.json -// 10. Exit the process -async function bootstrap() { - const buildManifest = await loadBuildManifest(); - - if (typeof process.env.TRIGGER_API_URL !== "string") { - console.error("TRIGGER_API_URL is not set"); - process.exit(1); - } - - const cliApiClient = new CliApiClient( - process.env.TRIGGER_API_URL, - process.env.TRIGGER_SECRET_KEY - ); - - if (!process.env.TRIGGER_PROJECT_REF) { - console.error("TRIGGER_PROJECT_REF is not set"); - process.exit(1); - } - - if (!process.env.TRIGGER_DEPLOYMENT_ID) { - console.error("TRIGGER_DEPLOYMENT_ID is not set"); - process.exit(1); - } - - return { - buildManifest, - cliApiClient, - projectRef: process.env.TRIGGER_PROJECT_REF, - deploymentId: process.env.TRIGGER_DEPLOYMENT_ID, - }; -} - -type BootstrapResult = Awaited>; - -async function indexDeployment({ - cliApiClient, - projectRef, - deploymentId, - buildManifest, -}: BootstrapResult) { - try { - const env = await cliApiClient.getEnvironmentVariables(projectRef); - - if (!env.success) { - throw new Error(`Failed to fetch environment variables: ${env.error}`); - } - - injectEnvVars(env.data.variables); - - const { config } = await importConfig(buildManifest.configPath); - - // This needs to run or the PrismaInstrumentation will throw an error - new TracingSDK({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", - instrumentations: config.instrumentations ?? [], - diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", - forceFlushTimeoutMillis: 30_000, - }); - - const importErrors: Array<{ error: Error; file: string }> = []; - - for (const file of buildManifest.files) { - const [error, module] = await $import(file.out); - - if (error) { - importErrors.push({ error, file: file.entry }); - - continue; - } - - for (const exportName of getExportNames(module)) { - const task = module[exportName] ?? module.default?.[exportName]; - - if (!task) { - continue; - } - - if (task[Symbol.for("trigger.dev/task")]) { - if (taskCatalog.taskExists(task.id)) { - taskCatalog.registerTaskFileMetadata(task.id, { - exportName, - filePath: file.entry, - entryPoint: file.out, - }); - } - } - } - } - - console.log("Import errors", importErrors); - - if (importErrors.length > 0) { - const errorMessages = importErrors.map((error) => { - return `${error.file}: ${error.error.message}`; - }); - - throw new Error(`Failed to index task files:\n${errorMessages.join("\n")}`); - } - - let tasks = taskCatalog.listTaskManifests(); - - if (typeof config.machine === "string") { - // Set the machine preset on all tasks that don't have it - tasks = tasks.map((task) => { - if (typeof task.machine?.preset !== "string") { - return { - ...task, - machine: { - preset: config.machine, - }, - }; - } - - return task; - }); - } - - const workerManifest: WorkerManifest = { - tasks, - configPath: buildManifest.configPath, - runtime: buildManifest.runtime, - executorEntryPoint: buildManifest.executorEntryPoint, - workerEntryPoint: buildManifest.workerEntryPoint, - loaderEntryPoint: buildManifest.loaderEntryPoint, - otelImportHook: buildManifest.otelImportHook, - }; - - console.log("Writing index.json", process.cwd()); - - await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2)); - - const sourceFiles = resolveTaskSourceFiles(buildManifest.sources, workerManifest.tasks); - - const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = { - localOnly: true, - metadata: { - contentHash: buildManifest.contentHash, - packageVersion: buildManifest.packageVersion, - cliPackageVersion: buildManifest.cliPackageVersion, - tasks: workerManifest.tasks, - sourceFiles, - }, - supportsLazyAttempts: true, - }; - - await cliApiClient.createDeploymentBackgroundWorker(deploymentId, backgroundWorkerBody); - } catch (error) { - // If we have an error, we need to fail the deployment - await cliApiClient.failDeployment(deploymentId, { - error: - error instanceof Error - ? { - name: error.name, - message: error.message, - stack: error.stack, - } - : { - name: "Error", - message: String(error), - }, - }); - - throw error; - } -} - -const results = await bootstrap(); - -await indexDeployment(results); - -function getExportNames(module: any) { - const exports: string[] = []; - - const exportKeys = Object.keys(module); - - if (exportKeys.length === 0) { - return exports; - } - - if (exportKeys.length === 1 && exportKeys[0] === "default") { - return Object.keys(module.default); - } - - return exportKeys; -} - -type Result = [Error | null, T | null]; - -async function $import(path: string): Promise> { - try { - const module = await import(path); - - return [null, module]; - } catch (error) { - return [error as Error, null]; - } -} - -function injectEnvVars(env: Record) { - for (const [key, value] of Object.entries(env)) { - process.env[key] = value; - } -} diff --git a/packages/cli-v3/src/entryPoints/deploy.ts b/packages/cli-v3/src/entryPoints/deploy-run-controller.ts similarity index 97% rename from packages/cli-v3/src/entryPoints/deploy.ts rename to packages/cli-v3/src/entryPoints/deploy-run-controller.ts index 6fb585a75..f5def22ef 100644 --- a/packages/cli-v3/src/entryPoints/deploy.ts +++ b/packages/cli-v3/src/entryPoints/deploy-run-controller.ts @@ -31,13 +31,14 @@ import { TaskRunProcess, } from "../executions/taskRunProcess.js"; import { checkpointSafeTimeout, unboundedTimeout } from "@trigger.dev/core/v3/utils/timers"; +import { env } from "std-env"; -const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber()); -const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1"; -const COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080); -const MACHINE_NAME = process.env.MACHINE_NAME || "local"; -const POD_NAME = process.env.POD_NAME || "some-pod"; -const SHORT_HASH = process.env.TRIGGER_CONTENT_HASH!.slice(0, 9); +const HTTP_SERVER_PORT = Number(env.HTTP_SERVER_PORT || getRandomPortNumber()); +const COORDINATOR_HOST = env.COORDINATOR_HOST || "127.0.0.1"; +const COORDINATOR_PORT = Number(env.COORDINATOR_PORT || 50080); +const MACHINE_NAME = env.MACHINE_NAME || "local"; +const POD_NAME = env.POD_NAME || "some-pod"; +const SHORT_HASH = env.TRIGGER_CONTENT_HASH!.slice(0, 9); const logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`); @@ -48,17 +49,17 @@ const defaultBackoff = new ExponentialBackoff("FullJitter", { cliLogger.loggerLevel = "debug"; cliLogger.debug("Starting prod worker", { - env: process.env, + env, }); class ProdWorker { - private contentHash = process.env.TRIGGER_CONTENT_HASH!; - private projectRef = process.env.TRIGGER_PROJECT_REF!; - private envId = process.env.TRIGGER_ENV_ID!; - private runId = process.env.TRIGGER_RUN_ID!; - private deploymentId = process.env.TRIGGER_DEPLOYMENT_ID!; - private deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION!; - private runningInKubernetes = !!process.env.KUBERNETES_PORT; + private contentHash = env.TRIGGER_CONTENT_HASH!; + private projectRef = env.TRIGGER_PROJECT_REF!; + private envId = env.TRIGGER_ENV_ID!; + private runId = env.TRIGGER_RUN_ID!; + private deploymentId = env.TRIGGER_DEPLOYMENT_ID!; + private deploymentVersion = env.TRIGGER_DEPLOYMENT_VERSION!; + private runningInKubernetes = !!env.KUBERNETES_PORT; private executing = false; private completed = new Set(); @@ -1291,15 +1292,15 @@ const prodWorker = new ProdWorker(HTTP_SERVER_PORT, workerManifest); await prodWorker.start(); function gatherProcessEnv(): Record { - const env = { - NODE_ENV: process.env.NODE_ENV ?? "production", - NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, - OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + const $env = { + NODE_ENV: env.NODE_ENV ?? "production", + NODE_EXTRA_CA_CERTS: env.NODE_EXTRA_CA_CERTS, + OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", }; // Filter out undefined values return Object.fromEntries( - Object.entries(env).filter(([key, value]) => value !== undefined) + Object.entries($env).filter(([key, value]) => value !== undefined) ) as Record; } diff --git a/packages/cli-v3/src/entryPoints/deploy-executor.ts b/packages/cli-v3/src/entryPoints/deploy-run-worker.ts similarity index 97% rename from packages/cli-v3/src/entryPoints/deploy-executor.ts rename to packages/cli-v3/src/entryPoints/deploy-run-worker.ts index 988f7cc87..b4e424741 100644 --- a/packages/cli-v3/src/entryPoints/deploy-executor.ts +++ b/packages/cli-v3/src/entryPoints/deploy-run-worker.ts @@ -35,6 +35,7 @@ import { readFile } from "node:fs/promises"; import sourceMapSupport from "source-map-support"; import { VERSION } from "../version.js"; import { setTimeout, setInterval } from "node:timers/promises"; +import { env } from "std-env"; sourceMapSupport.install({ handleUncaughtExceptions: false, @@ -119,9 +120,9 @@ async function bootstrap() { const { config, handleError } = await importConfig(workerManifest.configPath); const tracingSDK = new TracingSDK({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", instrumentations: config.instrumentations ?? [], - diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", forceFlushTimeoutMillis: 30_000, }); @@ -353,7 +354,7 @@ async function flushTracingSDK(timeoutInMs: number = 10_000) { } const prodRuntimeManager = new ProdRuntimeManager(zodIpc, { - waitThresholdInMs: parseInt(process.env.TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS ?? "30000", 10), + waitThresholdInMs: parseInt(env.TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS ?? "30000", 10), }); runtime.setGlobalRuntimeManager(prodRuntimeManager); diff --git a/packages/cli-v3/src/entryPoints/dev-index-worker.ts b/packages/cli-v3/src/entryPoints/dev-index-worker.ts new file mode 100644 index 000000000..55500eb66 --- /dev/null +++ b/packages/cli-v3/src/entryPoints/dev-index-worker.ts @@ -0,0 +1,139 @@ +import { + BuildManifest, + type HandleErrorFunction, + indexerToWorkerMessages, + taskCatalog, + TriggerConfig, +} from "@trigger.dev/core/v3"; +import { + StandardTaskCatalog, + TracingDiagnosticLogLevel, + TracingSDK, +} from "@trigger.dev/core/v3/workers"; +import { sendMessageInCatalog, ZodSchemaParsedError } from "@trigger.dev/core/v3/zodMessageHandler"; +import { readFile } from "node:fs/promises"; +import sourceMapSupport from "source-map-support"; +import { registerTasks } from "../indexing/registerTasks.js"; +import { env } from "std-env"; + +sourceMapSupport.install({ + handleUncaughtExceptions: false, + environment: "node", + hookRequire: false, +}); + +process.on("uncaughtException", function (error, origin) { + if (error instanceof Error) { + process.send && + process.send({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }); + } else { + process.send && + process.send({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), + }, + origin, + }, + version: "v1", + }); + } +}); + +taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog()); + +async function importConfig( + configPath: string +): Promise<{ config: TriggerConfig; handleError?: HandleErrorFunction }> { + const configModule = await import(configPath); + + const config = configModule?.default ?? configModule?.config; + + return { + config, + handleError: configModule?.handleError, + }; +} + +async function loadBuildManifest() { + const manifestContents = await readFile(env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8"); + const raw = JSON.parse(manifestContents); + + return BuildManifest.parse(raw); +} + +async function bootstrap() { + const buildManifest = await loadBuildManifest(); + + const { config } = await importConfig(buildManifest.configPath); + + // This needs to run or the PrismaInstrumentation will throw an error + const tracingSDK = new TracingSDK({ + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + instrumentations: config.instrumentations ?? [], + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + forceFlushTimeoutMillis: 30_000, + }); + + const importErrors = await registerTasks(buildManifest); + + return { + tracingSDK, + config, + buildManifest, + importErrors, + }; +} + +const { buildManifest, importErrors } = await bootstrap(); + +const tasks = taskCatalog.listTaskManifests(); + +await sendMessageInCatalog( + indexerToWorkerMessages, + "INDEX_COMPLETE", + { + manifest: { + tasks, + configPath: buildManifest.configPath, + runtime: buildManifest.runtime, + workerEntryPoint: buildManifest.runWorkerEntryPoint, + controllerEntryPoint: buildManifest.runControllerEntryPoint, + loaderEntryPoint: buildManifest.loaderEntryPoint, + }, + importErrors, + }, + async (msg) => { + process.send?.(msg); + } +).catch((err) => { + if (err instanceof ZodSchemaParsedError) { + return sendMessageInCatalog( + indexerToWorkerMessages, + "TASKS_FAILED_TO_PARSE", + { zodIssues: err.error.issues, tasks }, + async (msg) => { + await process.send?.(msg); + } + ); + } else { + console.error("Failed to send TASKS_READY message", err); + } + + return; +}); + +await new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 10); +}); diff --git a/packages/cli-v3/src/entryPoints/dev-executor.ts b/packages/cli-v3/src/entryPoints/dev-run-worker.ts similarity index 97% rename from packages/cli-v3/src/entryPoints/dev-executor.ts rename to packages/cli-v3/src/entryPoints/dev-run-worker.ts index 18febd3d4..ea8021d78 100644 --- a/packages/cli-v3/src/entryPoints/dev-executor.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-worker.ts @@ -33,6 +33,7 @@ import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc"; import { readFile } from "node:fs/promises"; import sourceMapSupport from "source-map-support"; import { VERSION } from "../version.js"; +import { env } from "std-env"; sourceMapSupport.install({ handleUncaughtExceptions: false, @@ -90,7 +91,7 @@ async function importConfig( } async function loadWorkerManifest() { - const manifestContents = await readFile(process.env.TRIGGER_WORKER_MANIFEST_PATH!, "utf-8"); + const manifestContents = await readFile(env.TRIGGER_WORKER_MANIFEST_PATH!, "utf-8"); const raw = JSON.parse(manifestContents); return WorkerManifest.parse(raw); @@ -102,9 +103,9 @@ async function bootstrap() { const { config, handleError } = await importConfig(workerManifest.configPath); const tracingSDK = new TracingSDK({ - url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", + url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", instrumentations: config.instrumentations ?? [], - diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", + diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", forceFlushTimeoutMillis: 30_000, }); diff --git a/packages/cli-v3/src/executions/errors.ts b/packages/cli-v3/src/executions/errors.ts deleted file mode 100644 index 265caea88..000000000 --- a/packages/cli-v3/src/executions/errors.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { z } from "zod"; - -export class UncaughtExceptionError extends Error { - constructor( - public readonly originalError: { name: string; message: string; stack?: string }, - public readonly origin: "uncaughtException" | "unhandledRejection" - ) { - super(`Uncaught exception: ${originalError.message}`); - - this.name = "UncaughtExceptionError"; - } -} - -export class TaskMetadataParseError extends Error { - constructor( - public readonly zodIssues: z.ZodIssue[], - public readonly tasks: any - ) { - super(`Failed to parse task metadata`); - - this.name = "TaskMetadataParseError"; - } -} - -export class UnexpectedExitError extends Error { - constructor( - public code: number, - public signal: NodeJS.Signals | null, - public stderr: string | undefined - ) { - super(`Unexpected exit with code ${code}`); - - this.name = "UnexpectedExitError"; - } -} - -export class CleanupProcessError extends Error { - constructor() { - super("Cancelled"); - - this.name = "CleanupProcessError"; - } -} - -export class CancelledProcessError extends Error { - constructor() { - super("Cancelled"); - - this.name = "CancelledProcessError"; - } -} - -export class SigKillTimeoutProcessError extends Error { - constructor() { - super("Process kill timeout"); - - this.name = "SigKillTimeoutProcessError"; - } -} - -export class GracefulExitTimeoutError extends Error { - constructor() { - super("Graceful exit timeout"); - - this.name = "GracefulExitTimeoutError"; - } -} - -export function getFriendlyErrorMessage( - code: number, - signal: NodeJS.Signals | null, - stderr: string | undefined, - dockerMode = true -) { - const message = (text: string) => { - if (signal) { - return `[${signal}] ${text}`; - } else { - return text; - } - }; - - if (code === 137) { - if (dockerMode) { - return message( - "Process ran out of memory! Try choosing a machine preset with more memory for this task." - ); - } else { - // Note: containerState reason and message should be checked to clarify the error - return message( - "Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task." - ); - } - } - - if (stderr?.includes("OOMErrorHandler")) { - return message( - "Process ran out of memory! Try choosing a machine preset with more memory for this task." - ); - } - - return message(`Process exited with code ${code}.`); -} diff --git a/packages/cli-v3/src/executions/taskRunProcess.ts b/packages/cli-v3/src/executions/taskRunProcess.ts index c62d56bb8..3d1ba3a52 100644 --- a/packages/cli-v3/src/executions/taskRunProcess.ts +++ b/packages/cli-v3/src/executions/taskRunProcess.ts @@ -22,9 +22,9 @@ import { CancelledProcessError, CleanupProcessError, GracefulExitTimeoutError, - SigKillTimeoutProcessError, UnexpectedExitError, -} from "./errors.js"; +} from "@trigger.dev/core/v3/errors"; +import { env } from "std-env"; export type OnWaitForDurationMessage = InferSocketMessageSchema< typeof ExecutorToWorkerMessageCatalog, @@ -105,18 +105,18 @@ export class TaskRunProcess { } async initialize() { - const { env, workerManifest, cwd, messageId } = this.options; + const { env: $env, workerManifest, cwd, messageId } = this.options; const fullEnv = { ...(this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {}), - ...env, + ...$env, OTEL_IMPORT_HOOK_INCLUDES: workerManifest.otelImportHook?.include?.join(","), // TODO: this will probably need to use something different for bun (maybe --preload?) NODE_OPTIONS: workerManifest.loaderEntryPoint ? `--import=${workerManifest.loaderEntryPoint} ${ - env.NODE_OPTIONS ?? process.env.NODE_OPTIONS ?? "" + env.NODE_OPTIONS ?? env.NODE_OPTIONS ?? "" }` - : env.NODE_OPTIONS ?? process.env.NODE_OPTIONS ?? "", + : env.NODE_OPTIONS ?? env.NODE_OPTIONS ?? "", }; logger.debug(`[${this.runId}] initializing task run process`, { @@ -125,7 +125,7 @@ export class TaskRunProcess { cwd, }); - this._child = fork(workerManifest.executorEntryPoint, executorArgs(workerManifest), { + this._child = fork(workerManifest.workerEntryPoint, executorArgs(workerManifest), { stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"], cwd, env: fullEnv, diff --git a/packages/cli-v3/src/indexing/indexWorkerManifest.ts b/packages/cli-v3/src/indexing/indexWorkerManifest.ts new file mode 100644 index 000000000..c8af3dac3 --- /dev/null +++ b/packages/cli-v3/src/indexing/indexWorkerManifest.ts @@ -0,0 +1,118 @@ +import { execPathForRuntime } from "@trigger.dev/core/v3/build"; +import { + BuildRuntime, + indexerToWorkerMessages, + WorkerManifest, +} from "@trigger.dev/core/v3/schemas"; +import { parseMessageFromCatalog } from "@trigger.dev/core/v3/zodMessageHandler"; +import { fork } from "node:child_process"; +import { env } from "std-env"; +import { + TaskIndexingImportError, + TaskMetadataParseError, + UncaughtExceptionError, +} from "@trigger.dev/core/v3/errors"; + +export type IndexWorkerManifestOptions = { + runtime: BuildRuntime; + indexWorkerPath: string; + buildManifestPath: string; + nodeOptions?: string; + env: Record; + cwd?: string; + otelHookInclude?: string[]; + otelHookExclude?: string[]; + handleStdout?: (data: string) => void; + handleStderr?: (data: string) => void; +}; + +export async function indexWorkerManifest({ + runtime, + indexWorkerPath, + buildManifestPath, + nodeOptions, + env: $env, + cwd, + otelHookInclude, + otelHookExclude, + handleStderr, + handleStdout, +}: IndexWorkerManifestOptions) { + return await new Promise((resolve, reject) => { + let resolved = false; + + const child = fork(indexWorkerPath, { + stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"], + cwd, + env: { + ...$env, + OTEL_IMPORT_HOOK_INCLUDES: otelHookInclude?.join(","), + OTEL_IMPORT_HOOK_EXCLUDES: otelHookExclude?.join(","), + TRIGGER_BUILD_MANIFEST_PATH: buildManifestPath, + NODE_OPTIONS: nodeOptions ? `${nodeOptions} ${env.NODE_OPTIONS ?? ""}` : env.NODE_OPTIONS, + }, + execPath: execPathForRuntime(runtime), + }); + + // Set a timeout to kill the child process if it doesn't respond + const timeout = setTimeout(() => { + if (resolved) { + return; + } + + resolved = true; + child.kill(); + reject(new Error("Worker timed out")); + }, 20_000); + + child.on("message", async (msg: any) => { + const message = parseMessageFromCatalog(msg, indexerToWorkerMessages); + + switch (message.type) { + case "INDEX_COMPLETE": { + clearTimeout(timeout); + resolved = true; + if (message.payload.importErrors.length > 0) { + reject( + new TaskIndexingImportError(message.payload.importErrors, message.payload.manifest) + ); + } else { + resolve(message.payload.manifest); + } + child.kill(); + break; + } + case "TASKS_FAILED_TO_PARSE": { + clearTimeout(timeout); + resolved = true; + reject(new TaskMetadataParseError(message.payload.zodIssues, message.payload.tasks)); + child.kill(); + break; + } + case "UNCAUGHT_EXCEPTION": { + clearTimeout(timeout); + resolved = true; + reject(new UncaughtExceptionError(message.payload.error, message.payload.origin)); + child.kill(); + break; + } + } + }); + + child.on("exit", (code) => { + if (!resolved) { + clearTimeout(timeout); + resolved = true; + reject(new Error(`Worker exited with code ${code}`)); + } + }); + + child.stdout?.on("data", (data) => { + handleStdout?.(data.toString()); + }); + + child.stderr?.on("data", (data) => { + handleStderr?.(data.toString()); + }); + }); +} diff --git a/packages/cli-v3/src/indexing/registerTasks.ts b/packages/cli-v3/src/indexing/registerTasks.ts new file mode 100644 index 000000000..21fc48dc0 --- /dev/null +++ b/packages/cli-v3/src/indexing/registerTasks.ts @@ -0,0 +1,68 @@ +import { BuildManifest, ImportTaskFileErrors, taskCatalog } from "@trigger.dev/core/v3"; + +export async function registerTasks(buildManifest: BuildManifest): Promise { + const importErrors: ImportTaskFileErrors = []; + + for (const file of buildManifest.files) { + const [error, module] = await tryImport(file.out); + + if (error) { + importErrors.push({ + file: file.entry, + message: error.message, + stack: error.stack, + name: error.name, + }); + + continue; + } + + for (const exportName of getExportNames(module)) { + const task = module[exportName] ?? module.default?.[exportName]; + + if (!task) { + continue; + } + + if (task[Symbol.for("trigger.dev/task")]) { + if (taskCatalog.taskExists(task.id)) { + taskCatalog.registerTaskFileMetadata(task.id, { + exportName, + filePath: file.entry, + entryPoint: file.out, + }); + } + } + } + } + + return importErrors; +} + +type Result = [Error | null, T | null]; + +async function tryImport(path: string): Promise> { + try { + const module = await import(path); + + return [null, module]; + } catch (error) { + return [error as Error, null]; + } +} + +function getExportNames(module: any) { + const exports: string[] = []; + + const exportKeys = Object.keys(module); + + if (exportKeys.length === 0) { + return exports; + } + + if (exportKeys.length === 1 && exportKeys[0] === "default") { + return Object.keys(module.default); + } + + return exportKeys; +} diff --git a/packages/cli-v3/src/package.json b/packages/cli-v3/src/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/packages/cli-v3/src/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/packages/cli-v3/src/telemetry/tracing.ts b/packages/cli-v3/src/telemetry/tracing.ts index 68d53eb58..e00e0668b 100644 --- a/packages/cli-v3/src/telemetry/tracing.ts +++ b/packages/cli-v3/src/telemetry/tracing.ts @@ -10,18 +10,19 @@ import { } from "@opentelemetry/semantic-conventions"; import { logger } from "../utilities/logger.js"; import { VERSION } from "../version.js"; +import { env } from "std-env"; function initializeTracing(): NodeTracerProvider | undefined { if ( process.argv.includes("--skip-telemetry") || - process.env.TRIGGER_DEV_SKIP_TELEMETRY || // only for backwards compat - process.env.TRIGGER_TELEMETRY_DISABLED + env.TRIGGER_DEV_SKIP_TELEMETRY || // only for backwards compat + env.TRIGGER_TELEMETRY_DISABLED ) { logger.debug("📉 Telemetry disabled"); return; } - if (process.env.OTEL_INTERNAL_DIAG_DEBUG) { + if (env.OTEL_INTERNAL_DIAG_DEBUG) { diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); } diff --git a/packages/cli-v3/src/utilities/cliOutput.ts b/packages/cli-v3/src/utilities/cliOutput.ts index 18069e77d..0827d846f 100644 --- a/packages/cli-v3/src/utilities/cliOutput.ts +++ b/packages/cli-v3/src/utilities/cliOutput.ts @@ -1,8 +1,9 @@ import { log } from "@clack/prompts"; import chalk from "chalk"; import terminalLink, { Options as TerminalLinkOptions } from "terminal-link"; +import { hasTTY } from "std-env"; -export const isInteractive = process.stdin.isTTY; +export const isInteractive = hasTTY; export const green = "#4FFF54"; export const purple = "#735BF3"; @@ -81,7 +82,7 @@ export function prettyError(header: string, body?: string, footer?: string) { .map((line) => `${indent}${line}`) .join("\n"); - const prettyBody = withIndents(body); + const prettyBody = withIndents(body?.trim()); const prettyFooter = withIndents(footer); log.error( diff --git a/packages/cli-v3/src/utilities/dotEnv.ts b/packages/cli-v3/src/utilities/dotEnv.ts index 616908c9c..33e29c7a7 100644 --- a/packages/cli-v3/src/utilities/dotEnv.ts +++ b/packages/cli-v3/src/utilities/dotEnv.ts @@ -1,5 +1,6 @@ import dotenv from "dotenv"; import { resolve } from "node:path"; +import { env } from "std-env"; export function resolveDotEnvVars(cwd?: string) { const result: { [key: string]: string } = {}; @@ -11,7 +12,7 @@ export function resolveDotEnvVars(cwd?: string) { ), }); - process.env.TRIGGER_API_URL && (result.TRIGGER_API_URL = process.env.TRIGGER_API_URL); + env.TRIGGER_API_URL && (result.TRIGGER_API_URL = env.TRIGGER_API_URL); // remove TRIGGER_API_URL and TRIGGER_SECRET_KEY, since those should be coming from the worker delete result.TRIGGER_API_URL; diff --git a/packages/cli-v3/src/utilities/eventBus.ts b/packages/cli-v3/src/utilities/eventBus.ts index 9fc6869b6..0865fbea4 100644 --- a/packages/cli-v3/src/utilities/eventBus.ts +++ b/packages/cli-v3/src/utilities/eventBus.ts @@ -1,4 +1,9 @@ -import { BuildTarget, TaskRunExecutionPayload, TaskRunExecutionResult } from "@trigger.dev/core/v3"; +import { + BuildManifest, + BuildTarget, + TaskRunExecutionPayload, + TaskRunExecutionResult, +} from "@trigger.dev/core/v3"; import { EventEmitter } from "node:events"; import { BackgroundWorker } from "../dev/backgroundWorker.js"; @@ -7,6 +12,7 @@ export type EventBusEvents = { buildStarted: [BuildTarget]; workerSkipped: []; backgroundWorkerInitialized: [BackgroundWorker]; + backgroundWorkerIndexingError: [BuildManifest, Error]; runStarted: [BackgroundWorker, TaskRunExecutionPayload]; runCompleted: [BackgroundWorker, TaskRunExecutionPayload, TaskRunExecutionResult, number]; }; diff --git a/packages/cli-v3/src/utilities/getEnvironmentVariableFactory.ts b/packages/cli-v3/src/utilities/getEnvironmentVariableFactory.ts deleted file mode 100644 index 35419f701..000000000 --- a/packages/cli-v3/src/utilities/getEnvironmentVariableFactory.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { logger } from "./logger.js"; - -type VariableNames = "TRIGGER_API_URL" | "TRIGGER_SECRET_KEY" | "TRIGGER_LOG_LEVEL"; - -type DeprecatedNames = ""; - -/** - * Create a function used to access an environment variable. - * - * This is not memoized to allow us to change the value at runtime, such as in testing. - * A warning is shown if the client is using a deprecated version - but only once. - */ -export function getEnvironmentVariableFactory({ - variableName, - deprecatedName, -}: { - variableName: VariableNames; - deprecatedName?: DeprecatedNames; -}): () => string | undefined; - -/** - * Create a function used to access an environment variable, with a default value. - * - * This is not memoized to allow us to change the value at runtime, such as in testing. - * A warning is shown if the client is using a deprecated version - but only once. - */ -export function getEnvironmentVariableFactory({ - variableName, - deprecatedName, - defaultValue, -}: { - variableName: VariableNames; - deprecatedName?: DeprecatedNames; - defaultValue: () => string; -}): () => string; - -/** - * Create a function used to access an environment variable. - * - * This is not memoized to allow us to change the value at runtime, such as in testing. - * A warning is shown if the client is using a deprecated version - but only once. - */ -export function getEnvironmentVariableFactory({ - variableName, - deprecatedName, - defaultValue, -}: { - variableName: VariableNames; - deprecatedName?: DeprecatedNames; - defaultValue?: () => string; -}): () => string | undefined { - let hasWarned = false; - return () => { - if (process.env[variableName]) { - return process.env[variableName]; - } else if (deprecatedName && process.env[deprecatedName]) { - if (!hasWarned) { - // Only show the warning once. - hasWarned = true; - logger.warn( - `Using "${deprecatedName}" environment variable. This is deprecated. Please use "${variableName}", instead.` - ); - } - return process.env[deprecatedName]; - } else { - return defaultValue?.(); - } - }; -} diff --git a/packages/cli-v3/src/utilities/logger.ts b/packages/cli-v3/src/utilities/logger.ts index 66f4fdb0e..640ceaa48 100644 --- a/packages/cli-v3/src/utilities/logger.ts +++ b/packages/cli-v3/src/utilities/logger.ts @@ -5,7 +5,8 @@ import chalk from "chalk"; import CLITable from "cli-table3"; import { formatMessagesSync } from "esbuild"; import type { Message } from "esbuild"; -import { getEnvironmentVariableFactory } from "./getEnvironmentVariableFactory.js"; +import { env } from "std-env"; + export const LOGGER_LEVELS = { none: -1, error: 0, @@ -26,12 +27,8 @@ const LOGGER_LEVEL_FORMAT_TYPE_MAP = { debug: undefined, } as const; -const getLogLevelFromEnv = getEnvironmentVariableFactory({ - variableName: "TRIGGER_LOG_LEVEL", -}); - function getLoggerLevel(): LoggerLevel { - const fromEnv = getLogLevelFromEnv()?.toLowerCase(); + const fromEnv = env.TRIGGER_LOG_LEVEL?.toLowerCase(); if (fromEnv !== undefined) { if (fromEnv in LOGGER_LEVELS) return fromEnv as LoggerLevel; const expected = Object.keys(LOGGER_LEVELS) diff --git a/packages/cli-v3/src/utilities/windows.ts b/packages/cli-v3/src/utilities/windows.ts index e2783c91a..4e8a6ad79 100644 --- a/packages/cli-v3/src/utilities/windows.ts +++ b/packages/cli-v3/src/utilities/windows.ts @@ -1,6 +1,7 @@ import { log, spinner as clackSpinner } from "@clack/prompts"; +import { isWindows as stdEnvIsWindows } from "std-env"; -export const isWindows = process.platform === "win32"; +export const isWindows = stdEnvIsWindows; export function escapeImportPath(path: string) { return isWindows ? path.replaceAll("\\", "\\\\") : path; diff --git a/packages/core/src/v3/errors.ts b/packages/core/src/v3/errors.ts index 6b965e72c..a29157af7 100644 --- a/packages/core/src/v3/errors.ts +++ b/packages/core/src/v3/errors.ts @@ -1,5 +1,8 @@ import { z } from "zod"; -import { TaskRunError } from "./schemas/common.js"; +import { DeploymentErrorData } from "./schemas/api.js"; +import { ImportTaskFileErrors, WorkerManifest } from "./schemas/build.js"; +import { SerializedError, TaskRunError } from "./schemas/common.js"; +import { TaskMetadataFailedToParseData } from "./schemas/messages.js"; export class AbortTaskRunError extends Error { constructor(message: string) { @@ -60,14 +63,6 @@ export function createErrorTaskError(error: TaskRunError): any { } } -export const SerializedError = z.object({ - message: z.string(), - name: z.string().optional(), - stackTrace: z.string().optional(), -}); - -export type SerializedError = z.infer; - export function createJsonErrorObject(error: TaskRunError): SerializedError { switch (error.type) { case "BUILT_IN_ERROR": { @@ -178,7 +173,7 @@ export function groupTaskMetadataIssuesByTask(tasks: any, issues: z.ZodIssue[]) return acc; } - const taskIndex = issue.path[1]; + const taskIndex = issue.path[2]; if (typeof taskIndex !== "number") { return acc; @@ -190,7 +185,7 @@ export function groupTaskMetadataIssuesByTask(tasks: any, issues: z.ZodIssue[]) return acc; } - const restOfPath = issue.path.slice(2); + const restOfPath = issue.path.slice(3); const taskId = task.id; const taskName = task.exportName; @@ -226,3 +221,272 @@ export function groupTaskMetadataIssuesByTask(tasks: any, issues: z.ZodIssue[]) > ); } + +export class UncaughtExceptionError extends Error { + constructor( + public readonly originalError: { name: string; message: string; stack?: string }, + public readonly origin: "uncaughtException" | "unhandledRejection" + ) { + super(`Uncaught exception: ${originalError.message}`); + + this.name = "UncaughtExceptionError"; + } +} + +export class TaskMetadataParseError extends Error { + constructor( + public readonly zodIssues: z.ZodIssue[], + public readonly tasks: any + ) { + super(`Failed to parse task metadata`); + + this.name = "TaskMetadataParseError"; + } +} + +export class TaskIndexingImportError extends Error { + constructor( + public readonly importErrors: ImportTaskFileErrors, + public readonly manifest: WorkerManifest + ) { + super(`Failed to import some task files`); + + this.name = "TaskIndexingImportError"; + } +} + +export class UnexpectedExitError extends Error { + constructor( + public code: number, + public signal: NodeJS.Signals | null, + public stderr: string | undefined + ) { + super(`Unexpected exit with code ${code}`); + + this.name = "UnexpectedExitError"; + } +} + +export class CleanupProcessError extends Error { + constructor() { + super("Cancelled"); + + this.name = "CleanupProcessError"; + } +} + +export class CancelledProcessError extends Error { + constructor() { + super("Cancelled"); + + this.name = "CancelledProcessError"; + } +} + +export class SigKillTimeoutProcessError extends Error { + constructor() { + super("Process kill timeout"); + + this.name = "SigKillTimeoutProcessError"; + } +} + +export class GracefulExitTimeoutError extends Error { + constructor() { + super("Graceful exit timeout"); + + this.name = "GracefulExitTimeoutError"; + } +} + +export function getFriendlyErrorMessage( + code: number, + signal: NodeJS.Signals | null, + stderr: string | undefined, + dockerMode = true +) { + const message = (text: string) => { + if (signal) { + return `[${signal}] ${text}`; + } else { + return text; + } + }; + + if (code === 137) { + if (dockerMode) { + return message( + "Process ran out of memory! Try choosing a machine preset with more memory for this task." + ); + } else { + // Note: containerState reason and message should be checked to clarify the error + return message( + "Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task." + ); + } + } + + if (stderr?.includes("OOMErrorHandler")) { + return message( + "Process ran out of memory! Try choosing a machine preset with more memory for this task." + ); + } + + return message(`Process exited with code ${code}.`); +} + +export function serializeIndexingError(error: unknown, stderr?: string): DeploymentErrorData { + if (error instanceof TaskMetadataParseError) { + return { + name: "TaskMetadataParseError", + message: "There was an error parsing the task metadata", + stack: JSON.stringify({ zodIssues: error.zodIssues, tasks: error.tasks }), + stderr, + }; + } else if (error instanceof TaskIndexingImportError) { + return { + name: "TaskIndexingImportError", + message: "There was an error importing task files", + stack: JSON.stringify(error.importErrors), + stderr, + }; + } else if (error instanceof UncaughtExceptionError) { + const originalError = error.originalError; + + return { + name: originalError.name, + message: originalError.message, + stack: originalError.stack, + stderr, + }; + } else if (error instanceof Error) { + return { + name: error.name, + message: error.message, + stack: error.stack, + stderr, + }; + } + + return { + name: "UnknownError", + message: String(error), + stderr, + }; +} + +export function prepareDeploymentError( + errorData: DeploymentErrorData +): DeploymentErrorData | undefined { + if (!errorData) { + return; + } + + if (errorData.name === "TaskMetadataParseError") { + const errorJson = tryJsonParse(errorData.stack); + + if (errorJson) { + const parsedError = TaskMetadataFailedToParseData.safeParse(errorJson); + + if (parsedError.success) { + return { + name: errorData.name, + message: errorData.message, + stack: createTaskMetadataFailedErrorStack(parsedError.data), + stderr: errorData.stderr, + }; + } else { + return { + name: errorData.name, + message: errorData.message, + stderr: errorData.stderr, + }; + } + } else { + return { + name: errorData.name, + message: errorData.message, + stderr: errorData.stderr, + }; + } + } else if (errorData.name === "TaskIndexingImportError") { + const errorJson = tryJsonParse(errorData.stack); + + if (errorJson) { + const parsedError = ImportTaskFileErrors.safeParse(errorJson); + + if (parsedError.success) { + return { + name: errorData.name, + message: errorData.message, + stack: parsedError.data + .map((error) => { + return `x ${error.message} in ${error.file}`; + }) + .join("\n"), + stderr: errorData.stderr, + }; + } else { + return { + name: errorData.name, + message: errorData.message, + stderr: errorData.stderr, + }; + } + } else { + return { + name: errorData.name, + message: errorData.message, + stderr: errorData.stderr, + }; + } + } + + return { + name: errorData.name, + message: errorData.message, + stack: errorData.stack, + stderr: errorData.stderr, + }; +} + +export function createTaskMetadataFailedErrorStack( + data: z.infer +): string { + const stack = []; + + const groupedIssues = groupTaskMetadataIssuesByTask(data.tasks, data.zodIssues); + + for (const key in groupedIssues) { + const taskWithIssues = groupedIssues[key]; + + if (!taskWithIssues) { + continue; + } + + stack.push("\n"); + stack.push(` ❯ ${taskWithIssues.exportName} in ${taskWithIssues.filePath}`); + + for (const issue of taskWithIssues.issues) { + if (issue.path) { + stack.push(` x ${issue.path} ${issue.message}`); + } else { + stack.push(` x ${issue.message}`); + } + } + } + + return stack.join("\n"); +} + +function tryJsonParse(data: string | undefined): any { + if (!data) { + return; + } + + try { + return JSON.parse(data); + } catch { + return; + } +} diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index cd4f93647..ee9388e18 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -1,7 +1,7 @@ import { z } from "zod"; -import { BackgroundWorkerMetadata, ImageDetailsMetadata } from "./resources.js"; +import { BackgroundWorkerMetadata } from "./resources.js"; import { QueueOptions } from "./schemas.js"; -import { SerializedError } from "../errors.js"; +import { SerializedError } from "./common.js"; export const WhoAmIResponseSchema = z.object({ userId: z.string(), @@ -195,6 +195,8 @@ export const DeploymentErrorData = z.object({ stderr: z.string().optional(), }); +export type DeploymentErrorData = z.infer; + export const FailDeploymentRequestBody = z.object({ error: DeploymentErrorData, }); diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index d1a571402..fd3ca6f2a 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -33,9 +33,10 @@ export const BuildManifest = z.object({ }) ), outputPath: z.string(), - indexerEntryPoint: z.string(), - executorEntryPoint: z.string(), - workerEntryPoint: z.string().optional(), + runWorkerEntryPoint: z.string(), // Dev & Deploy has a runWorkerEntryPoint + runControllerEntryPoint: z.string().optional(), // Only deploy has a runControllerEntryPoint + indexWorkerEntryPoint: z.string(), // Dev & Deploy has a indexWorkerEntryPoint + indexControllerEntryPoint: z.string().optional(), // Only deploy has a indexControllerEntryPoint loaderEntryPoint: z.string().optional(), configPath: z.string(), externals: BuildExternal.array().optional(), @@ -73,8 +74,8 @@ export type IndexMessage = z.infer; export const WorkerManifest = z.object({ configPath: z.string(), tasks: TaskManifest.array(), - executorEntryPoint: z.string(), - workerEntryPoint: z.string().optional(), + workerEntryPoint: z.string(), + controllerEntryPoint: z.string().optional(), loaderEntryPoint: z.string().optional(), runtime: BuildRuntime, otelImportHook: z @@ -95,3 +96,16 @@ export const WorkerManifestMessage = z.object({ }); export type WorkerManifestMessage = z.infer; + +export const ImportError = z.object({ + message: z.string(), + file: z.string(), + stack: z.string().optional(), + name: z.string().optional(), +}); + +export type ImportError = z.infer; + +export const ImportTaskFileErrors = z.array(ImportError); + +export type ImportTaskFileErrors = z.infer; diff --git a/packages/core/src/v3/schemas/common.ts b/packages/core/src/v3/schemas/common.ts index 06ac0eb72..575d7f506 100644 --- a/packages/core/src/v3/schemas/common.ts +++ b/packages/core/src/v3/schemas/common.ts @@ -280,3 +280,11 @@ export const BatchTaskRunExecutionResult = z.object({ }); export type BatchTaskRunExecutionResult = z.infer; + +export const SerializedError = z.object({ + message: z.string(), + name: z.string().optional(), + stackTrace: z.string().optional(), +}); + +export type SerializedError = z.infer; diff --git a/packages/core/src/v3/schemas/messages.ts b/packages/core/src/v3/schemas/messages.ts index 900fa2905..5ce65de9f 100644 --- a/packages/core/src/v3/schemas/messages.ts +++ b/packages/core/src/v3/schemas/messages.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { WorkerManifest } from "./build.js"; +import { ImportTaskFileErrors, WorkerManifest } from "./build.js"; import { MachinePreset, TaskRunExecution, @@ -11,7 +11,6 @@ import { EnvironmentType, ProdTaskRunExecution, ProdTaskRunExecutionPayload, - TaskManifest, TaskRunExecutionLazyAttemptPayload, WaitReason, } from "./schemas.js"; @@ -129,6 +128,7 @@ export const indexerToWorkerMessages = { INDEX_COMPLETE: z.object({ version: z.literal("v1").default("v1"), manifest: WorkerManifest, + importErrors: ImportTaskFileErrors, }), TASKS_FAILED_TO_PARSE: TaskMetadataFailedToParseData, UNCAUGHT_EXCEPTION: UncaughtExceptionMessage, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7eda6333d..f81b0e972 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -978,6 +978,9 @@ importers: source-map-support: specifier: 0.5.21 version: 0.5.21 + std-env: + specifier: ^3.7.0 + version: 3.7.0 terminal-link: specifier: ^3.0.0 version: 3.0.0 @@ -1360,6 +1363,9 @@ importers: '@sindresorhus/slugify': specifier: ^2.2.1 version: 2.2.1 + '@t3-oss/env-core': + specifier: ^0.11.0 + version: 0.11.0(typescript@5.5.4)(zod@3.23.8) '@t3-oss/env-nextjs': specifier: ^0.10.1 version: 0.10.1(typescript@5.5.4)(zod@3.23.8) @@ -12287,6 +12293,19 @@ packages: zod: 3.23.8 dev: false + /@t3-oss/env-core@0.11.0(typescript@5.5.4)(zod@3.23.8): + resolution: {integrity: sha512-PSalC5bG0a7XbyoLydiQdAnx3gICX6IQNctvh+TyLrdFxsxgocdj9Ui7sd061UlBzi+z4aIGjnem1kZx9QtUgQ==} + peerDependencies: + typescript: '>=5.0.0' + zod: ^3.0.0 + peerDependenciesMeta: + typescript: + optional: true + dependencies: + typescript: 5.5.4 + zod: 3.23.8 + dev: false + /@t3-oss/env-nextjs@0.10.1(typescript@5.5.4)(zod@3.23.8): resolution: {integrity: sha512-iy2qqJLnFh1RjEWno2ZeyTu0ufomkXruUsOZludzDIroUabVvHsrSjtkHqwHp1/pgPUzN3yBRHMILW162X7x2Q==} peerDependencies: @@ -25489,13 +25508,8 @@ packages: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} - /std-env@3.3.2: - resolution: {integrity: sha512-uUZI65yrV2Qva5gqE0+A7uVAvO40iPo6jGhs7s8keRfHCmtg+uB2X6EiLGCI9IgL1J17xGhvoOqSz79lzICPTA==} - dev: true - /std-env@3.7.0: resolution: {integrity: sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==} - dev: true /stoppable@1.1.0: resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==} @@ -27761,7 +27775,7 @@ packages: pathe: 1.1.0 picocolors: 1.0.0 source-map: 0.6.1 - std-env: 3.3.2 + std-env: 3.7.0 strip-literal: 1.0.1 tinybench: 2.3.1 tinypool: 0.3.1 diff --git a/references/v3-catalog/package.json b/references/v3-catalog/package.json index 8e8968837..29cdae99a 100644 --- a/references/v3-catalog/package.json +++ b/references/v3-catalog/package.json @@ -23,6 +23,7 @@ "@react-email/components": "^0.0.17", "@react-email/render": "^0.0.7", "@sindresorhus/slugify": "^2.2.1", + "@t3-oss/env-core": "^0.11.0", "@t3-oss/env-nextjs": "^0.10.1", "@traceloop/instrumentation-openai": "^0.10.0", "@trigger.dev/sdk": "workspace:*", diff --git a/references/v3-catalog/src/env.ts b/references/v3-catalog/src/env.ts new file mode 100644 index 000000000..ddacb5bd8 --- /dev/null +++ b/references/v3-catalog/src/env.ts @@ -0,0 +1,13 @@ +import { createEnv } from "@t3-oss/env-core"; +import { z } from "zod"; + +export const env = createEnv({ + server: { + UPLOADTHING_SECRET: z.string(), + UPLOADTHING_APP_ID: z.string(), + OPENAI_API_KEY: z.string(), + }, + + runtimeEnv: process.env, + emptyStringAsUndefined: true, +}); diff --git a/references/v3-catalog/src/trigger/openai.mts b/references/v3-catalog/src/trigger/openai.mts index 97db420e1..1cbc5dc7e 100644 --- a/references/v3-catalog/src/trigger/openai.mts +++ b/references/v3-catalog/src/trigger/openai.mts @@ -1,9 +1,10 @@ +import { env } from "@/env.js"; import { logger, task } from "@trigger.dev/sdk/v3"; import OpenAI from "openai"; const openai = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, + apiKey: env.OPENAI_API_KEY, }); export const openaiTask = task({