github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Improve and unify the indexing between dev and deploy

Browse Source
This commit is contained in:
Eric Allam
2024-08-21 13:09:10 +01:00
committed by Eric Allam
parent 61d9fd3384
commit ac222106b2
43 changed files with 1153 additions and 819 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/launch.json
Vendored
+1 -1
View File
@@ -41,7 +41,7 @@
"type": "node-terminal",
"request": "launch",
"name": "Debug V3 Deploy CLI",
"command": "pnpm exec triggerdev deploy --self-hosted",
"command": "pnpm exec triggerdev deploy --self-hosted --load-image",
"cwd": "${workspaceFolder}/references/v3-catalog",
"sourceMaps": true
},
apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts
+5 -71
View File
@@ -1,16 +1,13 @@
import {
DeploymentErrorData,
ExternalBuildData,
TaskMetadataFailedToParseData,
groupTaskMetadataIssuesByTask,
prepareDeploymentError,
} from "@trigger.dev/core/v3";
import { WorkerDeployment, WorkerDeploymentStatus } from "@trigger.dev/database";
import { z } from "zod";
import { WorkerDeployment } from "@trigger.dev/database";
import { PrismaClient, prisma } from "~/db.server";
import { Organization } from "~/models/organization.server";
import { Project } from "~/models/project.server";
import { User } from "~/models/user.server";
import { safeJsonParse } from "~/utils/json";
import { getUsername } from "~/utils/username";
export type ErrorData = {
@@ -164,75 +161,12 @@ export class DeploymentPresenter {
return;
}
const parsedErrorData = DeploymentErrorData.safeParse(errorData);
const deploymentError = DeploymentErrorData.safeParse(errorData);
if (!parsedErrorData.success) {
if (!deploymentError.success) {
return;
}
if (parsedErrorData.data.name === "TaskMetadataParseError") {
const errorJson = safeJsonParse(parsedErrorData.data.stack);
if (errorJson) {
const parsedError = TaskMetadataFailedToParseData.safeParse(errorJson);
if (parsedError.success) {
return {
name: parsedErrorData.data.name,
message: parsedErrorData.data.message,
stack: createTaskMetadataFailedErrorStack(parsedError.data),
stderr: parsedErrorData.data.stderr,
};
} else {
return {
name: parsedErrorData.data.name,
message: parsedErrorData.data.message,
stderr: parsedErrorData.data.stderr,
};
}
} else {
return {
name: parsedErrorData.data.name,
message: parsedErrorData.data.message,
stderr: parsedErrorData.data.stderr,
};
}
}
return {
name: parsedErrorData.data.name,
message: parsedErrorData.data.message,
stack: parsedErrorData.data.stack,
stderr: parsedErrorData.data.stderr,
};
return prepareDeploymentError(deploymentError.data);
}
}
function createTaskMetadataFailedErrorStack(
data: z.infer<typeof TaskMetadataFailedToParseData>
): string {
const stack = [];
const groupedIssues = groupTaskMetadataIssuesByTask(data.tasks, data.zodIssues);
for (const key in groupedIssues) {
const taskWithIssues = groupedIssues[key];
if (!taskWithIssues) {
continue;
}
stack.push("\n");
stack.push(` ❯ ${taskWithIssues.exportName} in ${taskWithIssues.filePath}`);
for (const issue of taskWithIssues.issues) {
if (issue.path) {
stack.push(` x ${issue.path} ${issue.message}`);
} else {
stack.push(` x ${issue.message}`);
}
}
}
return stack.join("\n");
}
packages/cli-v3/package.json
+1
View File
@@ -114,6 +114,7 @@
"resolve": "^1.22.8",
"signal-exit": "^4.1.0",
"source-map-support": "0.5.21",
"std-env": "^3.7.0",
"terminal-link": "^3.0.0",
"tiny-invariant": "^1.2.0",
"tinyexec": "^0.2.0",
packages/cli-v3/src/build/bundle.ts
+24 -18
View File
@@ -9,11 +9,12 @@ import {
deployEntryPoints,
devEntryPoints,
isConfigEntryPoint,
isExecutorEntryPointForTarget,
isIndexerEntryPointForTarget,
isRunWorkerForTarget,
isIndexWorkerForTarget,
isLoaderEntryPoint,
isWorkerEntryPointForTarget,
isRunControllerForTarget,
shims,
isIndexControllerForTarget,
} from "./packageModules.js";
import { buildPlugins } from "./plugins.js";
@@ -34,9 +35,10 @@ export type BundleResult = {
files: TaskFile[];
configPath: string;
loaderEntryPoint: string | undefined;
workerEntryPoint: string | undefined;
indexerEntryPoint: string | undefined;
executorEntryPoint: string | undefined;
runWorkerEntryPoint: string | undefined;
runControllerEntryPoint: string | undefined;
indexWorkerEntryPoint: string | undefined;
indexControllerEntryPoint: string | undefined;
stop: (() => Promise<void>) | undefined;
};
@@ -145,9 +147,10 @@ export async function getBundleResultFromBuild(
let configPath: string | undefined;
let loaderEntryPoint: string | undefined;
let workerEntryPoint: string | undefined;
let executorEntryPoint: string | undefined;
let indexerEntryPoint: string | undefined;
let runWorkerEntryPoint: string | undefined;
let runControllerEntryPoint: string | undefined;
let indexWorkerEntryPoint: string | undefined;
let indexControllerEntryPoint: string | undefined;
for (const [outputPath, outputMeta] of Object.entries(result.metafile.outputs)) {
if (outputPath.endsWith(".mjs")) {
@@ -161,12 +164,14 @@ export async function getBundleResultFromBuild(
configPath = $outputPath;
} else if (isLoaderEntryPoint(outputMeta.entryPoint)) {
loaderEntryPoint = $outputPath;
} else if (isWorkerEntryPointForTarget(outputMeta.entryPoint, target)) {
workerEntryPoint = $outputPath;
} else if (isIndexerEntryPointForTarget(outputMeta.entryPoint, target)) {
indexerEntryPoint = $outputPath;
} else if (isExecutorEntryPointForTarget(outputMeta.entryPoint, target)) {
executorEntryPoint = $outputPath;
} else if (isRunControllerForTarget(outputMeta.entryPoint, target)) {
runControllerEntryPoint = $outputPath;
} else if (isRunWorkerForTarget(outputMeta.entryPoint, target)) {
runWorkerEntryPoint = $outputPath;
} else if (isIndexControllerForTarget(outputMeta.entryPoint, target)) {
indexControllerEntryPoint = $outputPath;
} else if (isIndexWorkerForTarget(outputMeta.entryPoint, target)) {
indexWorkerEntryPoint = $outputPath;
} else {
if (
!outputMeta.entryPoint.startsWith("..") &&
@@ -189,9 +194,10 @@ export async function getBundleResultFromBuild(
files,
configPath: configPath,
loaderEntryPoint,
workerEntryPoint,
executorEntryPoint,
indexerEntryPoint,
runWorkerEntryPoint,
runControllerEntryPoint,
indexWorkerEntryPoint,
indexControllerEntryPoint,
contentHash: hasher.digest("hex"),
};
}
packages/cli-v3/src/build/manifests.ts
+8 -1
View File
@@ -17,7 +17,14 @@ export async function copyManifestToDir(
updatedManifest.configPath = updatedManifest.configPath.replace(source, destination);
updatedManifest.loaderEntryPoint = updatedManifest.loaderEntryPoint?.replace(source, destination);
updatedManifest.workerEntryPoint = updatedManifest.workerEntryPoint?.replace(source, destination);
updatedManifest.runWorkerEntryPoint = updatedManifest.runWorkerEntryPoint.replace(
source,
destination
);
updatedManifest.indexWorkerEntryPoint = updatedManifest.indexWorkerEntryPoint.replace(
source,
destination
);
updatedManifest.files = updatedManifest.files.map((file) => {
return {
packages/cli-v3/src/build/packageModules.ts
+46 -32
View File
@@ -2,20 +2,22 @@ import { join } from "node:path";
import { sourceDir } from "../sourceDir.js";
import { BuildTarget } from "@trigger.dev/core/v3";
export const devExecutorEntryPoint = join(sourceDir, "entryPoints", "dev-executor.js");
export const devIndexerEntryPoint = join(sourceDir, "entryPoints", "dev-indexer.js");
export const devRunWorker = join(sourceDir, "entryPoints", "dev-run-worker.js");
export const devIndexWorker = join(sourceDir, "entryPoints", "dev-index-worker.js");
export const deployEntryPoint = join(sourceDir, "entryPoints", "deploy.js");
export const deployExecutorEntryPoint = join(sourceDir, "entryPoints", "deploy-executor.js");
export const deployIndexerEntryPoint = join(sourceDir, "entryPoints", "deploy-indexer.js");
export const deployRunController = join(sourceDir, "entryPoints", "deploy-run-controller.js");
export const deployRunWorker = join(sourceDir, "entryPoints", "deploy-run-worker.js");
export const deployIndexController = join(sourceDir, "entryPoints", "deploy-index-controller.js");
export const deployIndexWorker = join(sourceDir, "entryPoints", "deploy-index-worker.js");
export const telemetryEntryPoint = join(sourceDir, "entryPoints", "loader.js");
export const devEntryPoints = [devExecutorEntryPoint, devIndexerEntryPoint, telemetryEntryPoint];
export const devEntryPoints = [devRunWorker, devIndexWorker, telemetryEntryPoint];
export const deployEntryPoints = [
deployIndexerEntryPoint,
deployExecutorEntryPoint,
deployEntryPoint,
deployRunController,
deployRunWorker,
deployIndexController,
deployIndexWorker,
telemetryEntryPoint,
];
@@ -23,41 +25,61 @@ export const esmShimPath = join(sourceDir, "shims", "esm.js");
export const shims = [esmShimPath];
function isDevExecutorEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-executor.js"));
function isDevRunWorker(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-run-worker.js"));
}
function isDevIndexerEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-indexer.js"));
function isDevIndexWorker(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "dev-index-worker.js"));
}
function isDeployIndexerEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-indexer.js"));
function isDeployIndexController(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-index-controller.js"));
}
function isDeployEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy.js"));
function isDeployIndexWorker(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-index-worker.js"));
}
function isDeployExecutorEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-executor.js"));
function isDeployRunController(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-run-controller.js"));
}
function isDeployRunWorker(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-run-worker.js"));
}
export function isLoaderEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "loader.js"));
}
export function isExecutorEntryPointForTarget(entryPoint: string, target: BuildTarget) {
export function isRunWorkerForTarget(entryPoint: string, target: BuildTarget) {
if (target === "dev") {
return isDevExecutorEntryPoint(entryPoint);
return isDevRunWorker(entryPoint);
} else {
return isDeployExecutorEntryPoint(entryPoint);
return isDeployRunWorker(entryPoint);
}
}
export function isWorkerEntryPointForTarget(entryPoint: string, target: BuildTarget) {
export function isRunControllerForTarget(entryPoint: string, target: BuildTarget) {
if (target === "deploy") {
return isDeployEntryPoint(entryPoint);
return isDeployRunController(entryPoint);
}
return false;
}
export function isIndexWorkerForTarget(entryPoint: string, target: BuildTarget) {
if (target === "dev") {
return isDevIndexWorker(entryPoint);
} else {
return isDeployIndexWorker(entryPoint);
}
}
export function isIndexControllerForTarget(entryPoint: string, target: BuildTarget) {
if (target === "deploy") {
return isDeployIndexController(entryPoint);
}
return false;
@@ -66,11 +88,3 @@ export function isWorkerEntryPointForTarget(entryPoint: string, target: BuildTar
export function isConfigEntryPoint(entryPoint: string) {
return entryPoint.startsWith("trigger.config.ts");
}
export function isIndexerEntryPointForTarget(entryPoint: string, target: BuildTarget) {
if (target === "dev") {
return isDevIndexerEntryPoint(entryPoint);
} else {
return isDeployIndexerEntryPoint(entryPoint);
}
}
packages/cli-v3/src/commands/deploy.ts
+108 -34
View File
@@ -1,5 +1,5 @@
import { intro, outro } from "@clack/prompts";
import { CORE_VERSION } from "@trigger.dev/core/v3";
import { CORE_VERSION, prepareDeploymentError } from "@trigger.dev/core/v3";
import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build";
import { BuildManifest, InitializeDeploymentResponseBody } from "@trigger.dev/core/v3/schemas";
import { Command, Option as CommandOption } from "commander";
@@ -16,10 +16,12 @@ import {
resolvePluginsForContext,
} from "../build/extensions.js";
import { createExternalsBuildExtension } from "../build/externals.js";
import { getInstrumentedPackageNames } from "../build/instrumentation.js";
import {
deployEntryPoint,
deployExecutorEntryPoint,
deployIndexerEntryPoint,
deployIndexController,
deployIndexWorker,
deployRunController,
deployRunWorker,
telemetryEntryPoint,
} from "../build/packageModules.js";
import {
@@ -39,7 +41,8 @@ import {
saveLogs,
} from "../deploy/logs.js";
import { buildManifestToJSON } from "../utilities/buildManifest.js";
import { chalkError, cliLink } from "../utilities/cliOutput.js";
import { chalkError, cliLink, prettyError } from "../utilities/cliOutput.js";
import { loadDotEnvVars } from "../utilities/dotEnv.js";
import { writeJSONFile } from "../utilities/fileSystem.js";
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
import { logger } from "../utilities/logger.js";
@@ -50,8 +53,6 @@ import { spinner } from "../utilities/windows.js";
import { VERSION } from "../version.js";
import { login } from "./login.js";
import { updateTriggerPackages } from "./update.js";
import { loadDotEnvVars, resolveDotEnvVars } from "../utilities/dotEnv.js";
import { getInstrumentedPackageNames } from "../build/instrumentation.js";
const DeployCommandOptions = CommonCommandOptions.extend({
dryRun: z.boolean().default(false),
@@ -250,9 +251,10 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
dirs: resolvedConfig.dirs,
},
outputPath: destination.path,
workerEntryPoint: bundleResult.workerEntryPoint ?? deployEntryPoint,
executorEntryPoint: bundleResult.executorEntryPoint ?? deployExecutorEntryPoint,
indexerEntryPoint: bundleResult.indexerEntryPoint ?? deployIndexerEntryPoint,
runControllerEntryPoint: bundleResult.runControllerEntryPoint ?? deployRunController,
runWorkerEntryPoint: bundleResult.runWorkerEntryPoint ?? deployRunWorker,
indexControllerEntryPoint: bundleResult.indexControllerEntryPoint ?? deployIndexController,
indexWorkerEntryPoint: bundleResult.indexWorkerEntryPoint ?? deployIndexWorker,
loaderEntryPoint: bundleResult.loaderEntryPoint ?? telemetryEntryPoint,
configPath: bundleResult.configPath,
deploy: {
@@ -484,11 +486,14 @@ function rewriteBuildManifestPaths(
})),
outputPath: rewriteOutputPath(destinationDir, buildManifest.outputPath),
configPath: rewriteOutputPath(destinationDir, buildManifest.configPath),
executorEntryPoint: rewriteOutputPath(destinationDir, buildManifest.executorEntryPoint),
indexerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.indexerEntryPoint),
workerEntryPoint: buildManifest.workerEntryPoint
? rewriteOutputPath(destinationDir, buildManifest.workerEntryPoint)
runControllerEntryPoint: buildManifest.runControllerEntryPoint
? rewriteOutputPath(destinationDir, buildManifest.runControllerEntryPoint)
: undefined,
runWorkerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.runWorkerEntryPoint),
indexControllerEntryPoint: buildManifest.indexControllerEntryPoint
? rewriteOutputPath(destinationDir, buildManifest.indexControllerEntryPoint)
: undefined,
indexWorkerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.indexWorkerEntryPoint),
loaderEntryPoint: buildManifest.loaderEntryPoint
? rewriteOutputPath(destinationDir, buildManifest.loaderEntryPoint)
: undefined,
@@ -550,15 +555,15 @@ function rewriteOutputPath(destinationDir: string, filePath: string) {
}
async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) {
if (!buildManifest.workerEntryPoint) {
throw new Error("No worker entry point found in build manifest");
if (!buildManifest.runControllerEntryPoint || !buildManifest.indexControllerEntryPoint) {
throw new Error("Something went wrong with the build. Aborting deployment. [code 7789]");
}
const containerfile = await generateContainerfile({
runtime: buildManifest.runtime,
workerEntryPoint: buildManifest.workerEntryPoint,
entrypoint: buildManifest.runControllerEntryPoint,
build: buildManifest.build,
indexerEntryPoint: buildManifest.indexerEntryPoint,
indexScript: buildManifest.indexControllerEntryPoint,
});
await writeFile(join(outputPath, "Containerfile"), containerfile);
@@ -589,25 +594,94 @@ async function failDeploy(
) {
$spinner.stop(`Failed to deploy project`);
// If there are logs, let's write it out to a temporary file and include the path in the error message
if (logs.trim() !== "") {
const logPath = await saveLogs(deployment.shortCode, logs);
const doOutputLogs = async (prefix: string = "Error") => {
if (logs.trim() !== "") {
const logPath = await saveLogs(deployment.shortCode, logs);
printWarnings(warnings);
printErrors(errors);
printWarnings(warnings);
printErrors(errors);
checkLogsForErrors(logs);
checkLogsForErrors(logs);
outro(
`${chalkError("Error:")} ${error.message}. Full build logs have been saved to ${logPath}`
);
outro(
`${chalkError(`${prefix}:`)} ${
error.message
}. Full build logs have been saved to ${logPath}`
);
} else {
outro(`${chalkError(`${prefix}:`)} ${error.message}.`);
}
};
const exitCommand = (message: string) => {
throw new SkipLoggingError(message);
};
const deploymentResponse = await client.getDeployment(deployment.id);
if (!deploymentResponse.success) {
logger.debug(`Failed to get deployment with worker: ${deploymentResponse.error}`);
} else {
outro(`${chalkError("Error:")} ${error.message}.`);
const serverDeployment = deploymentResponse.data;
switch (serverDeployment.status) {
case "PENDING":
case "DEPLOYING":
case "BUILDING": {
await doOutputLogs();
await client.failDeployment(deployment.id, {
error,
});
exitCommand("Failed to deploy project");
break;
}
case "CANCELED": {
await doOutputLogs("Canceled");
exitCommand("Failed to deploy project");
break;
}
case "FAILED": {
const errorData = serverDeployment.errorData
? prepareDeploymentError(serverDeployment.errorData)
: undefined;
if (errorData) {
prettyError(errorData.name, errorData.stack, errorData.stderr);
if (logs.trim() !== "") {
const logPath = await saveLogs(deployment.shortCode, logs);
outro(`Aborting deployment. Full build logs have been saved to ${logPath}`);
} else {
outro(`Aborting deployment`);
}
} else {
await doOutputLogs("Failed");
}
exitCommand("Failed to deploy project");
break;
}
case "DEPLOYED": {
await doOutputLogs("Deployed with errors");
exitCommand("Deployed with errors");
break;
}
case "TIMED_OUT": {
await doOutputLogs("TimedOut");
exitCommand("Timed out");
break;
}
}
}
await client.failDeployment(deployment.id, {
error,
});
throw new SkipLoggingError(`Failed to deploy: ${error.message}`);
}
packages/cli-v3/src/commands/dev.tsx
+6 -1
View File
@@ -12,6 +12,7 @@ import { logger } from "../utilities/logger.js";
import { runtimeChecks } from "../utilities/runtimeCheck.js";
import { getProjectClient, isLoggedIn, LoginResultOk } from "../utilities/session.js";
import { updateTriggerPackages } from "./update.js";
import { login } from "./login.js";
const DevCommandOptions = CommonCommandOptions.extend({
debugOtel: z.boolean().default(false),
@@ -45,7 +46,11 @@ export function configureDevCommand(program: Command) {
export async function devCommand(options: DevCommandOptions) {
runtimeChecks();
const authorization = await isLoggedIn(options.profile);
const authorization = await login({
embedded: true,
defaultApiUrl: options.apiUrl,
profile: options.profile,
});
if (!authorization.ok) {
if (authorization.error === "fetch failed") {
packages/cli-v3/src/commands/login.ts
+3 -2
View File
@@ -22,6 +22,7 @@ import { logger } from "../utilities/logger.js";
import { spinner } from "../utilities/windows.js";
import { isLinuxServer } from "../utilities/linux.js";
import { VERSION } from "../version.js";
import { env } from "std-env";
export const LoginCommandOptions = CommonCommandOptions.extend({
apiUrl: z.string(),
@@ -74,12 +75,12 @@ export async function login(options?: LoginOptions): Promise<LoginResult> {
intro("Logging in to Trigger.dev");
}
const accessTokenFromEnv = process.env.TRIGGER_ACCESS_TOKEN;
const accessTokenFromEnv = env.TRIGGER_ACCESS_TOKEN;
if (accessTokenFromEnv) {
const auth = {
accessToken: accessTokenFromEnv,
apiUrl: process.env.TRIGGER_API_URL ?? opts.defaultApiUrl ?? "https://api.trigger.dev",
apiUrl: env.TRIGGER_API_URL ?? opts.defaultApiUrl ?? "https://api.trigger.dev",
};
const apiClient = new CliApiClient(auth.apiUrl, auth.accessToken);
const userData = await apiClient.whoAmI();
packages/cli-v3/src/commands/update.ts
+2 -1
View File
@@ -11,6 +11,7 @@ import { printStandloneInitialBanner, updateCheck } from "../utilities/initialBa
import { logger } from "../utilities/logger.js";
import { spinner } from "../utilities/windows.js";
import { VERSION } from "../version.js";
import { hasTTY } from "std-env";
export const UpdateCommandOptions = CommonCommandOptions.pick({
logLevel: true,
@@ -142,7 +143,7 @@ export async function updateTriggerPackages(
);
}
if (!process.stdout.isTTY) {
if (!hasTTY) {
// Running in CI with version mismatch detected
outro("Deploy failed");
packages/cli-v3/src/deploy/buildImage.ts
+6 -6
View File
@@ -408,8 +408,8 @@ function extractImageDigest(outputs: string[]) {
export type GenerateContainerfileOptions = {
runtime: BuildRuntime;
build: BuildManifest["build"];
indexerEntryPoint: string;
workerEntryPoint: string;
indexScript: string;
entrypoint: string;
};
export async function generateContainerfile(options: GenerateContainerfileOptions) {
@@ -481,7 +481,7 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
NODE_ENV=production
# Run the indexer
RUN bun run ${options.indexerEntryPoint}
RUN bun run ${options.indexScript}
# Development or production stage builds upon the base stage
FROM base AS final
@@ -510,7 +510,7 @@ COPY --from=install --chown=bun:bun /app ./
# Copy the index.json file from the indexer stage
COPY --from=indexer --chown=bun:bun /app/index.json ./
ENTRYPOINT [ "dumb-init", "node", "${options.workerEntryPoint}" ]
ENTRYPOINT [ "dumb-init", "node", "${options.entrypoint}" ]
CMD []
`;
}
@@ -579,7 +579,7 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
NODE_OPTIONS="--max_old_space_size=8192"
# Run the indexer
RUN node ${options.indexerEntryPoint}
RUN node ${options.indexScript}
# Development or production stage builds upon the base stage
FROM base AS final
@@ -609,7 +609,7 @@ COPY --from=install --chown=node:node /app ./
# Copy the index.json file from the indexer stage
COPY --from=indexer --chown=node:node /app/index.json ./
ENTRYPOINT [ "dumb-init", "node", "${options.workerEntryPoint}" ]
ENTRYPOINT [ "dumb-init", "node", "${options.entrypoint}" ]
CMD []
`;
}
packages/cli-v3/src/dev/backgroundWorker.ts
+27 -82
View File
@@ -10,27 +10,23 @@ import {
TaskRunFailedExecutionResult,
WorkerManifest,
correctErrorStackTrace,
indexerToWorkerMessages,
} from "@trigger.dev/core/v3";
import { parseMessageFromCatalog } from "@trigger.dev/core/v3/zodMessageHandler";
import { Evt } from "evt";
import { fork } from "node:child_process";
import { execPathForRuntime } from "@trigger.dev/core/v3/build";
import { join } from "node:path";
import { TaskRunProcess, TaskRunProcessOptions } from "../executions/taskRunProcess.js";
import { eventBus } from "../utilities/eventBus.js";
import { writeJSONFile } from "../utilities/fileSystem.js";
import { logger } from "../utilities/logger.js";
import {
CancelledProcessError,
CleanupProcessError,
SigKillTimeoutProcessError,
TaskMetadataParseError,
UncaughtExceptionError,
UnexpectedExitError,
getFriendlyErrorMessage,
} from "../executions/errors.js";
} from "@trigger.dev/core/v3/errors";
import { TaskRunProcess, TaskRunProcessOptions } from "../executions/taskRunProcess.js";
import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js";
import { prettyError } from "../utilities/cliOutput.js";
import { eventBus } from "../utilities/eventBus.js";
import { writeJSONFile } from "../utilities/fileSystem.js";
import { logger } from "../utilities/logger.js";
export type CurrentWorkers = BackgroundWorkerCoordinator["currentWorkers"];
export class BackgroundWorkerCoordinator {
@@ -254,87 +250,36 @@ export class BackgroundWorker {
throw new Error("Worker already initialized");
}
let resolved = false;
// Write the build manifest to this.build.outputPath/build.json
await writeJSONFile(this.buildManifestPath, this.build, true);
logger.debug("Initializing worker", { build: this.build, params: this.params });
logger.debug("indexing worker manifest", { build: this.build, params: this.params });
this.manifest = await new Promise<WorkerManifest>((resolve, reject) => {
const child = fork(this.build.indexerEntryPoint, {
stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
cwd: this.params.cwd,
env: {
...this.params.env,
TRIGGER_BUILD_MANIFEST_PATH: this.buildManifestPath,
NODE_OPTIONS: this.build.loaderEntryPoint
? `--import=${this.build.loaderEntryPoint} ${process.env.NODE_OPTIONS ?? ""}`
: process.env.NODE_OPTIONS,
},
execPath: execPathForRuntime(this.build.runtime),
});
// Set a timeout to kill the child process if it doesn't respond
const timeout = setTimeout(() => {
if (resolved) {
return;
this.manifest = await indexWorkerManifest({
runtime: this.build.runtime,
indexWorkerPath: this.build.indexWorkerEntryPoint,
buildManifestPath: this.buildManifestPath,
nodeOptions: this.build.loaderEntryPoint
? `--import=${this.build.loaderEntryPoint}`
: undefined,
env: this.params.env,
cwd: this.params.cwd,
otelHookInclude: this.build.otelImportHook?.include,
otelHookExclude: this.build.otelImportHook?.exclude,
handleStdout(data) {
logger.debug(data);
},
handleStderr(data) {
if (!data.includes("Debugger attached")) {
prettyError(data.toString());
}
resolved = true;
child.kill();
reject(new Error("Worker timed out"));
}, 20_000);
child.on("message", async (msg: any) => {
const message = parseMessageFromCatalog(msg, indexerToWorkerMessages);
switch (message.type) {
case "INDEX_COMPLETE": {
clearTimeout(timeout);
resolved = true;
resolve(message.payload.manifest);
child.kill();
break;
}
case "TASKS_FAILED_TO_PARSE": {
clearTimeout(timeout);
resolved = true;
reject(new TaskMetadataParseError(message.payload.zodIssues, message.payload.tasks));
child.kill();
break;
}
case "UNCAUGHT_EXCEPTION": {
clearTimeout(timeout);
resolved = true;
reject(new UncaughtExceptionError(message.payload.error, message.payload.origin));
child.kill();
break;
}
}
});
child.on("exit", (code) => {
if (!resolved) {
clearTimeout(timeout);
resolved = true;
reject(new Error(`Worker exited with code ${code}`));
}
});
child.stdout?.on("data", (data) => {
logger.debug(`indexer: ${data.toString()}`);
});
child.stderr?.on("data", (data) => {
logger.debug(`indexer: ${data.toString()}`);
});
},
});
// Write the build manifest to this.build.outputPath/worker.json
await writeJSONFile(this.workerManifestPath, this.manifest, true);
logger.debug("Worker initialized", { path: this.build.outputPath });
logger.debug("worker manifest indexed", { path: this.build.outputPath });
}
// We need to notify all the task run processes that a task run has completed,
packages/cli-v3/src/dev/devOutput.ts
+36 -1
View File
@@ -11,11 +11,21 @@ import {
chalkWarning,
chalkWorker,
cliLink,
prettyError,
prettyPrintDate,
} from "../utilities/cliOutput.js";
import { eventBus, EventBusEventArgs } from "../utilities/eventBus.js";
import { TaskRunError, TaskRunErrorCodes } from "@trigger.dev/core/v3/schemas";
import {
TaskMetadataFailedToParseData,
TaskRunError,
TaskRunErrorCodes,
} from "@trigger.dev/core/v3/schemas";
import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
import {
createTaskMetadataFailedErrorStack,
TaskIndexingImportError,
TaskMetadataParseError,
} from "@trigger.dev/core/v3/errors";
export type DevOutputOptions = {
name: string | undefined;
@@ -63,6 +73,29 @@ export function startDevOutput(options: DevOutputOptions) {
);
};
const backgroundWorkerIndexingError = (
...[buildManifest, error]: EventBusEventArgs<"backgroundWorkerIndexingError">
) => {
if (error instanceof TaskIndexingImportError) {
for (const importError of error.importErrors) {
prettyError(`Could not import ${importError.file}`, importError.stack);
}
} else if (error instanceof TaskMetadataParseError) {
const errorStack = createTaskMetadataFailedErrorStack({
version: "v1",
zodIssues: error.zodIssues,
tasks: error.tasks,
});
prettyError(`Could not parse task metadata`, errorStack);
} else {
const errorText = error instanceof Error ? error.message : "Unknown error";
const stack = error instanceof Error ? error.stack : undefined;
prettyError(`Build failed: ${errorText}`, stack);
}
};
const runStarted = (...[worker, payload]: EventBusEventArgs<"runStarted">) => {
if (!worker.serverWorker) {
return;
@@ -142,6 +175,7 @@ export function startDevOutput(options: DevOutputOptions) {
eventBus.on("backgroundWorkerInitialized", backgroundWorkerInitialized);
eventBus.on("runStarted", runStarted);
eventBus.on("runCompleted", runCompleted);
eventBus.on("backgroundWorkerIndexingError", backgroundWorkerIndexingError);
return () => {
eventBus.off("rebuildStarted", rebuildStarted);
@@ -150,6 +184,7 @@ export function startDevOutput(options: DevOutputOptions) {
eventBus.off("backgroundWorkerInitialized", backgroundWorkerInitialized);
eventBus.off("runStarted", runStarted);
eventBus.off("runCompleted", runCompleted);
eventBus.off("backgroundWorkerIndexingError", backgroundWorkerIndexingError);
};
}
packages/cli-v3/src/dev/devSession.ts
+21 -17
View File
@@ -18,11 +18,7 @@ import {
} from "../build/extensions.js";
import { createExternalsBuildExtension } from "../build/externals.js";
import { copyManifestToDir } from "../build/manifests.js";
import {
devExecutorEntryPoint,
devIndexerEntryPoint,
telemetryEntryPoint,
} from "../build/packageModules.js";
import { devIndexWorker, devRunWorker, telemetryEntryPoint } from "../build/packageModules.js";
import { type DevCommandOptions } from "../commands/dev.js";
import { eventBus } from "../utilities/eventBus.js";
import { logger } from "../utilities/logger.js";
@@ -31,6 +27,7 @@ import { EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js";
import { VERSION } from "../version.js";
import { startDevOutput } from "./devOutput.js";
import { startWorkerRuntime } from "./workerRuntime.js";
import { getInstrumentedPackageNames } from "../build/instrumentation.js";
export type DevSessionOptions = {
name: string | undefined;
@@ -75,21 +72,25 @@ export async function startDevSession({
const pluginsFromExtensions = resolvePluginsForContext(buildContext);
async function updateBundle(bundle: BundleResult, workerDir?: EphemeralDirectory) {
let buildManifest = await createBuildManifestFromBundle(
bundle,
destination.path,
rawConfig,
workerDir?.path
);
buildManifest = await notifyExtensionOnBuildComplete(buildContext, buildManifest);
try {
let buildManifest = await createBuildManifestFromBundle(
bundle,
destination.path,
rawConfig,
workerDir?.path
);
buildManifest = await notifyExtensionOnBuildComplete(buildContext, buildManifest);
logger.debug("Updated bundle", { bundle, buildManifest });
await runtime.initializeWorker(buildManifest);
} catch (error) {
logger.error("Error updating bundle", { error });
if (error instanceof Error) {
eventBus.emit("backgroundWorkerIndexingError", buildManifest, error);
} else {
logger.error("Error updating bundle", { error });
}
}
}
@@ -191,14 +192,17 @@ async function createBuildManifestFromBundle(
dirs: resolvedConfig.dirs,
},
outputPath: destination,
executorEntryPoint: bundle.executorEntryPoint ?? devExecutorEntryPoint,
runWorkerEntryPoint: bundle.runWorkerEntryPoint ?? devRunWorker,
indexWorkerEntryPoint: bundle.indexWorkerEntryPoint ?? devIndexWorker,
loaderEntryPoint: bundle.loaderEntryPoint ?? telemetryEntryPoint,
indexerEntryPoint: bundle.indexerEntryPoint ?? devIndexerEntryPoint,
configPath: bundle.configPath,
deploy: {
env: {},
},
build: {},
otelImportHook: {
include: getInstrumentedPackageNames(resolvedConfig),
},
};
if (!workerDir) {
packages/cli-v3/src/dev/workerRuntime.ts
+4 -3
View File
@@ -26,6 +26,7 @@ import { eventBus } from "../utilities/eventBus.js";
import { logger } from "../utilities/logger.js";
import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js";
import { BackgroundWorker, BackgroundWorkerCoordinator } from "./backgroundWorker.js";
import { env } from "std-env";
export interface WorkerRuntime {
shutdown(): Promise<void>;
@@ -316,13 +317,13 @@ function WebsocketFactory(apiKey: string) {
}
function gatherProcessEnv() {
const env = {
...process.env,
const $env = {
...env,
NODE_ENV: "development",
};
// Filter out undefined values
return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined));
return Object.fromEntries(Object.entries($env).filter(([key, value]) => value !== undefined));
}
function validateWorkerManifest(manifest: WorkerManifest): string[] {
packages/cli-v3/src/entryPoints/deploy-index-controller.ts
+116
View File
@@ -0,0 +1,116 @@
import {
BuildManifest,
CreateBackgroundWorkerRequestBody,
serializeIndexingError,
} from "@trigger.dev/core/v3";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { env } from "std-env";
import { CliApiClient } from "../apiClient.js";
import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js";
import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js";
async function loadBuildManifest() {
const manifestContents = await readFile("./build.json", "utf-8");
const raw = JSON.parse(manifestContents);
return BuildManifest.parse(raw);
}
async function bootstrap() {
const buildManifest = await loadBuildManifest();
if (typeof env.TRIGGER_API_URL !== "string") {
console.error("TRIGGER_API_URL is not set");
process.exit(1);
}
const cliApiClient = new CliApiClient(env.TRIGGER_API_URL, env.TRIGGER_SECRET_KEY);
if (!env.TRIGGER_PROJECT_REF) {
console.error("TRIGGER_PROJECT_REF is not set");
process.exit(1);
}
if (!env.TRIGGER_DEPLOYMENT_ID) {
console.error("TRIGGER_DEPLOYMENT_ID is not set");
process.exit(1);
}
return {
buildManifest,
cliApiClient,
projectRef: env.TRIGGER_PROJECT_REF,
deploymentId: env.TRIGGER_DEPLOYMENT_ID,
};
}
type BootstrapResult = Awaited<ReturnType<typeof bootstrap>>;
async function indexDeployment({
cliApiClient,
projectRef,
deploymentId,
buildManifest,
}: BootstrapResult) {
const stdout: string[] = [];
const stderr: string[] = [];
try {
const $env = await cliApiClient.getEnvironmentVariables(projectRef);
if (!$env.success) {
throw new Error(`Failed to fetch environment variables: ${env.error}`);
}
const workerManifest = await indexWorkerManifest({
runtime: buildManifest.runtime,
indexWorkerPath: buildManifest.indexWorkerEntryPoint,
buildManifestPath: "./build.json",
nodeOptions: buildManifest.loaderEntryPoint
? `--import=${buildManifest.loaderEntryPoint}`
: undefined,
env: $env.data.variables,
otelHookExclude: buildManifest.otelImportHook?.exclude,
otelHookInclude: buildManifest.otelImportHook?.include,
handleStdout(data) {
stdout.push(data);
},
handleStderr(data) {
if (!data.includes("DeprecationWarning")) {
stderr.push(data);
}
},
});
console.log("Writing index.json", process.cwd());
await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2));
const sourceFiles = resolveTaskSourceFiles(buildManifest.sources, workerManifest.tasks);
const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = {
localOnly: true,
metadata: {
contentHash: buildManifest.contentHash,
packageVersion: buildManifest.packageVersion,
cliPackageVersion: buildManifest.cliPackageVersion,
tasks: workerManifest.tasks,
sourceFiles,
},
supportsLazyAttempts: true,
};
await cliApiClient.createDeploymentBackgroundWorker(deploymentId, backgroundWorkerBody);
} catch (error) {
const serialiedIndexError = serializeIndexingError(error, stderr.join("\n"));
await cliApiClient.failDeployment(deploymentId, { error: serialiedIndexError });
process.exit(1);
}
}
const results = await bootstrap();
await indexDeployment(results);
packages/cli-v3/src/entryPoints/dev-indexer.ts → packages/cli-v3/src/entryPoints/deploy-index-worker.ts
+11 -43
View File
@@ -13,6 +13,8 @@ import {
import { sendMessageInCatalog, ZodSchemaParsedError } from "@trigger.dev/core/v3/zodMessageHandler";
import { readFile } from "node:fs/promises";
import sourceMapSupport from "source-map-support";
import { registerTasks } from "../indexing/registerTasks.js";
import { env } from "std-env";
sourceMapSupport.install({
handleUncaughtExceptions: false,
@@ -63,7 +65,7 @@ async function importConfig(
}
async function loadBuildManifest() {
const manifestContents = await readFile(process.env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8");
const manifestContents = await readFile(env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8");
const raw = JSON.parse(manifestContents);
return BuildManifest.parse(raw);
@@ -76,42 +78,23 @@ async function bootstrap() {
// This needs to run or the PrismaInstrumentation will throw an error
const tracingSDK = new TracingSDK({
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
instrumentations: config.instrumentations ?? [],
diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
});
for (const file of buildManifest.files) {
const module = await import(file.out);
for (const exportName of getExportNames(module)) {
const task = module[exportName] ?? module.default?.[exportName];
if (!task) {
continue;
}
if (task[Symbol.for("trigger.dev/task")]) {
if (taskCatalog.taskExists(task.id)) {
taskCatalog.registerTaskFileMetadata(task.id, {
exportName,
filePath: file.entry,
entryPoint: file.out,
});
}
}
}
}
const importErrors = await registerTasks(buildManifest);
return {
tracingSDK,
config,
buildManifest,
importErrors,
};
}
const { buildManifest } = await bootstrap();
const { buildManifest, importErrors } = await bootstrap();
const tasks = taskCatalog.listTaskManifests();
@@ -123,10 +106,11 @@ await sendMessageInCatalog(
tasks,
configPath: buildManifest.configPath,
runtime: buildManifest.runtime,
executorEntryPoint: buildManifest.executorEntryPoint,
workerEntryPoint: buildManifest.workerEntryPoint,
workerEntryPoint: buildManifest.runWorkerEntryPoint,
controllerEntryPoint: buildManifest.runControllerEntryPoint,
loaderEntryPoint: buildManifest.loaderEntryPoint,
},
importErrors,
},
async (msg) => {
process.send?.(msg);
@@ -153,19 +137,3 @@ await new Promise<void>((resolve) => {
resolve();
}, 10);
});
function getExportNames(module: any) {
const exports: string[] = [];
const exportKeys = Object.keys(module);
if (exportKeys.length === 0) {
return exports;
}
if (exportKeys.length === 1 && exportKeys[0] === "default") {
return Object.keys(module.default);
}
return exportKeys;
}
packages/cli-v3/src/entryPoints/deploy-indexer.ts
-261
View File
@@ -1,261 +0,0 @@
import {
BuildManifest,
CreateBackgroundWorkerRequestBody,
type HandleErrorFunction,
taskCatalog,
TriggerConfig,
WorkerManifest,
} from "@trigger.dev/core/v3";
import {
StandardTaskCatalog,
TracingDiagnosticLogLevel,
TracingSDK,
} from "@trigger.dev/core/v3/workers";
import { readFile, writeFile } from "node:fs/promises";
import sourceMapSupport from "source-map-support";
import { CliApiClient } from "../apiClient.js";
import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js";
import { join } from "node:path";
sourceMapSupport.install({
handleUncaughtExceptions: false,
environment: "node",
hookRequire: false,
});
taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog());
async function importConfig(configPath: string): Promise<{
config: TriggerConfig;
handleError?: HandleErrorFunction;
}> {
const configModule = await import(configPath);
const config = configModule?.default ?? configModule?.config;
return {
config,
handleError: configModule?.handleError,
};
}
async function loadBuildManifest() {
const manifestContents = await readFile("./build.json", "utf-8");
const raw = JSON.parse(manifestContents);
return BuildManifest.parse(raw);
}
// We need to make sure, that if any errors are thrown, that we fail the deployment
// 1. Fetch the build manifest
// 2. Fetch the environment variables from the server
// 3. Import the config
// 5. Inject the env vars into process.env
// 6. Configure the tracing SDK
// 7. Load all the tasks from the build manifest and create the index.json
// 8. Write the index.json to the file system
// 9. Update the deployment with the worker index.json
// 10. Exit the process
async function bootstrap() {
const buildManifest = await loadBuildManifest();
if (typeof process.env.TRIGGER_API_URL !== "string") {
console.error("TRIGGER_API_URL is not set");
process.exit(1);
}
const cliApiClient = new CliApiClient(
process.env.TRIGGER_API_URL,
process.env.TRIGGER_SECRET_KEY
);
if (!process.env.TRIGGER_PROJECT_REF) {
console.error("TRIGGER_PROJECT_REF is not set");
process.exit(1);
}
if (!process.env.TRIGGER_DEPLOYMENT_ID) {
console.error("TRIGGER_DEPLOYMENT_ID is not set");
process.exit(1);
}
return {
buildManifest,
cliApiClient,
projectRef: process.env.TRIGGER_PROJECT_REF,
deploymentId: process.env.TRIGGER_DEPLOYMENT_ID,
};
}
type BootstrapResult = Awaited<ReturnType<typeof bootstrap>>;
async function indexDeployment({
cliApiClient,
projectRef,
deploymentId,
buildManifest,
}: BootstrapResult) {
try {
const env = await cliApiClient.getEnvironmentVariables(projectRef);
if (!env.success) {
throw new Error(`Failed to fetch environment variables: ${env.error}`);
}
injectEnvVars(env.data.variables);
const { config } = await importConfig(buildManifest.configPath);
// This needs to run or the PrismaInstrumentation will throw an error
new TracingSDK({
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
instrumentations: config.instrumentations ?? [],
diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
});
const importErrors: Array<{ error: Error; file: string }> = [];
for (const file of buildManifest.files) {
const [error, module] = await $import(file.out);
if (error) {
importErrors.push({ error, file: file.entry });
continue;
}
for (const exportName of getExportNames(module)) {
const task = module[exportName] ?? module.default?.[exportName];
if (!task) {
continue;
}
if (task[Symbol.for("trigger.dev/task")]) {
if (taskCatalog.taskExists(task.id)) {
taskCatalog.registerTaskFileMetadata(task.id, {
exportName,
filePath: file.entry,
entryPoint: file.out,
});
}
}
}
}
console.log("Import errors", importErrors);
if (importErrors.length > 0) {
const errorMessages = importErrors.map((error) => {
return `${error.file}: ${error.error.message}`;
});
throw new Error(`Failed to index task files:\n${errorMessages.join("\n")}`);
}
let tasks = taskCatalog.listTaskManifests();
if (typeof config.machine === "string") {
// Set the machine preset on all tasks that don't have it
tasks = tasks.map((task) => {
if (typeof task.machine?.preset !== "string") {
return {
...task,
machine: {
preset: config.machine,
},
};
}
return task;
});
}
const workerManifest: WorkerManifest = {
tasks,
configPath: buildManifest.configPath,
runtime: buildManifest.runtime,
executorEntryPoint: buildManifest.executorEntryPoint,
workerEntryPoint: buildManifest.workerEntryPoint,
loaderEntryPoint: buildManifest.loaderEntryPoint,
otelImportHook: buildManifest.otelImportHook,
};
console.log("Writing index.json", process.cwd());
await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2));
const sourceFiles = resolveTaskSourceFiles(buildManifest.sources, workerManifest.tasks);
const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = {
localOnly: true,
metadata: {
contentHash: buildManifest.contentHash,
packageVersion: buildManifest.packageVersion,
cliPackageVersion: buildManifest.cliPackageVersion,
tasks: workerManifest.tasks,
sourceFiles,
},
supportsLazyAttempts: true,
};
await cliApiClient.createDeploymentBackgroundWorker(deploymentId, backgroundWorkerBody);
} catch (error) {
// If we have an error, we need to fail the deployment
await cliApiClient.failDeployment(deploymentId, {
error:
error instanceof Error
? {
name: error.name,
message: error.message,
stack: error.stack,
}
: {
name: "Error",
message: String(error),
},
});
throw error;
}
}
const results = await bootstrap();
await indexDeployment(results);
function getExportNames(module: any) {
const exports: string[] = [];
const exportKeys = Object.keys(module);
if (exportKeys.length === 0) {
return exports;
}
if (exportKeys.length === 1 && exportKeys[0] === "default") {
return Object.keys(module.default);
}
return exportKeys;
}
type Result<T> = [Error | null, T | null];
async function $import(path: string): Promise<Result<any>> {
try {
const module = await import(path);
return [null, module];
} catch (error) {
return [error as Error, null];
}
}
function injectEnvVars(env: Record<string, string>) {
for (const [key, value] of Object.entries(env)) {
process.env[key] = value;
}
}
packages/cli-v3/src/entryPoints/deploy.ts → packages/cli-v3/src/entryPoints/deploy-run-controller.ts
+20 -19
View File
@@ -31,13 +31,14 @@ import {
TaskRunProcess,
} from "../executions/taskRunProcess.js";
import { checkpointSafeTimeout, unboundedTimeout } from "@trigger.dev/core/v3/utils/timers";
import { env } from "std-env";
const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
const COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
const POD_NAME = process.env.POD_NAME || "some-pod";
const SHORT_HASH = process.env.TRIGGER_CONTENT_HASH!.slice(0, 9);
const HTTP_SERVER_PORT = Number(env.HTTP_SERVER_PORT || getRandomPortNumber());
const COORDINATOR_HOST = env.COORDINATOR_HOST || "127.0.0.1";
const COORDINATOR_PORT = Number(env.COORDINATOR_PORT || 50080);
const MACHINE_NAME = env.MACHINE_NAME || "local";
const POD_NAME = env.POD_NAME || "some-pod";
const SHORT_HASH = env.TRIGGER_CONTENT_HASH!.slice(0, 9);
const logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
@@ -48,17 +49,17 @@ const defaultBackoff = new ExponentialBackoff("FullJitter", {
cliLogger.loggerLevel = "debug";
cliLogger.debug("Starting prod worker", {
env: process.env,
env,
});
class ProdWorker {
private contentHash = process.env.TRIGGER_CONTENT_HASH!;
private projectRef = process.env.TRIGGER_PROJECT_REF!;
private envId = process.env.TRIGGER_ENV_ID!;
private runId = process.env.TRIGGER_RUN_ID!;
private deploymentId = process.env.TRIGGER_DEPLOYMENT_ID!;
private deploymentVersion = process.env.TRIGGER_DEPLOYMENT_VERSION!;
private runningInKubernetes = !!process.env.KUBERNETES_PORT;
private contentHash = env.TRIGGER_CONTENT_HASH!;
private projectRef = env.TRIGGER_PROJECT_REF!;
private envId = env.TRIGGER_ENV_ID!;
private runId = env.TRIGGER_RUN_ID!;
private deploymentId = env.TRIGGER_DEPLOYMENT_ID!;
private deploymentVersion = env.TRIGGER_DEPLOYMENT_VERSION!;
private runningInKubernetes = !!env.KUBERNETES_PORT;
private executing = false;
private completed = new Set<string>();
@@ -1291,15 +1292,15 @@ const prodWorker = new ProdWorker(HTTP_SERVER_PORT, workerManifest);
await prodWorker.start();
function gatherProcessEnv(): Record<string, string> {
const env = {
NODE_ENV: process.env.NODE_ENV ?? "production",
NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS,
OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
const $env = {
NODE_ENV: env.NODE_ENV ?? "production",
NODE_EXTRA_CA_CERTS: env.NODE_EXTRA_CA_CERTS,
OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
};
// Filter out undefined values
return Object.fromEntries(
Object.entries(env).filter(([key, value]) => value !== undefined)
Object.entries($env).filter(([key, value]) => value !== undefined)
) as Record<string, string>;
}
packages/cli-v3/src/entryPoints/deploy-executor.ts → packages/cli-v3/src/entryPoints/deploy-run-worker.ts
+4 -3
View File
@@ -35,6 +35,7 @@ import { readFile } from "node:fs/promises";
import sourceMapSupport from "source-map-support";
import { VERSION } from "../version.js";
import { setTimeout, setInterval } from "node:timers/promises";
import { env } from "std-env";
sourceMapSupport.install({
handleUncaughtExceptions: false,
@@ -119,9 +120,9 @@ async function bootstrap() {
const { config, handleError } = await importConfig(workerManifest.configPath);
const tracingSDK = new TracingSDK({
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
instrumentations: config.instrumentations ?? [],
diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
});
@@ -353,7 +354,7 @@ async function flushTracingSDK(timeoutInMs: number = 10_000) {
}
const prodRuntimeManager = new ProdRuntimeManager(zodIpc, {
waitThresholdInMs: parseInt(process.env.TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS ?? "30000", 10),
waitThresholdInMs: parseInt(env.TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS ?? "30000", 10),
});
runtime.setGlobalRuntimeManager(prodRuntimeManager);
packages/cli-v3/src/entryPoints/dev-index-worker.ts
+139
View File
@@ -0,0 +1,139 @@
import {
BuildManifest,
type HandleErrorFunction,
indexerToWorkerMessages,
taskCatalog,
TriggerConfig,
} from "@trigger.dev/core/v3";
import {
StandardTaskCatalog,
TracingDiagnosticLogLevel,
TracingSDK,
} from "@trigger.dev/core/v3/workers";
import { sendMessageInCatalog, ZodSchemaParsedError } from "@trigger.dev/core/v3/zodMessageHandler";
import { readFile } from "node:fs/promises";
import sourceMapSupport from "source-map-support";
import { registerTasks } from "../indexing/registerTasks.js";
import { env } from "std-env";
sourceMapSupport.install({
handleUncaughtExceptions: false,
environment: "node",
hookRequire: false,
});
process.on("uncaughtException", function (error, origin) {
if (error instanceof Error) {
process.send &&
process.send({
type: "UNCAUGHT_EXCEPTION",
payload: {
error: { name: error.name, message: error.message, stack: error.stack },
origin,
},
version: "v1",
});
} else {
process.send &&
process.send({
type: "UNCAUGHT_EXCEPTION",
payload: {
error: {
name: "Error",
message: typeof error === "string" ? error : JSON.stringify(error),
},
origin,
},
version: "v1",
});
}
});
taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog());
async function importConfig(
configPath: string
): Promise<{ config: TriggerConfig; handleError?: HandleErrorFunction }> {
const configModule = await import(configPath);
const config = configModule?.default ?? configModule?.config;
return {
config,
handleError: configModule?.handleError,
};
}
async function loadBuildManifest() {
const manifestContents = await readFile(env.TRIGGER_BUILD_MANIFEST_PATH!, "utf-8");
const raw = JSON.parse(manifestContents);
return BuildManifest.parse(raw);
}
async function bootstrap() {
const buildManifest = await loadBuildManifest();
const { config } = await importConfig(buildManifest.configPath);
// This needs to run or the PrismaInstrumentation will throw an error
const tracingSDK = new TracingSDK({
url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
instrumentations: config.instrumentations ?? [],
diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
});
const importErrors = await registerTasks(buildManifest);
return {
tracingSDK,
config,
buildManifest,
importErrors,
};
}
const { buildManifest, importErrors } = await bootstrap();
const tasks = taskCatalog.listTaskManifests();
await sendMessageInCatalog(
indexerToWorkerMessages,
"INDEX_COMPLETE",
{
manifest: {
tasks,
configPath: buildManifest.configPath,
runtime: buildManifest.runtime,
workerEntryPoint: buildManifest.runWorkerEntryPoint,
controllerEntryPoint: buildManifest.runControllerEntryPoint,
loaderEntryPoint: buildManifest.loaderEntryPoint,
},
importErrors,
},
async (msg) => {
process.send?.(msg);
}
).catch((err) => {
if (err instanceof ZodSchemaParsedError) {
return sendMessageInCatalog(
indexerToWorkerMessages,
"TASKS_FAILED_TO_PARSE",
{ zodIssues: err.error.issues, tasks },
async (msg) => {
await process.send?.(msg);
}
);
} else {
console.error("Failed to send TASKS_READY message", err);
}
return;
});
await new Promise<void>((resolve) => {
setTimeout(() => {
resolve();
}, 10);
});
packages/cli-v3/src/entryPoints/dev-executor.ts → packages/cli-v3/src/entryPoints/dev-run-worker.ts
+4 -3
View File
@@ -33,6 +33,7 @@ import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
import { readFile } from "node:fs/promises";
import sourceMapSupport from "source-map-support";
import { VERSION } from "../version.js";
import { env } from "std-env";
sourceMapSupport.install({
handleUncaughtExceptions: false,
@@ -90,7 +91,7 @@ async function importConfig(
}
async function loadWorkerManifest() {
const manifestContents = await readFile(process.env.TRIGGER_WORKER_MANIFEST_PATH!, "utf-8");
const manifestContents = await readFile(env.TRIGGER_WORKER_MANIFEST_PATH!, "utf-8");
const raw = JSON.parse(manifestContents);
return WorkerManifest.parse(raw);
@@ -102,9 +103,9 @@ async function bootstrap() {
const { config, handleError } = await importConfig(workerManifest.configPath);
const tracingSDK = new TracingSDK({
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
instrumentations: config.instrumentations ?? [],
diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
});
packages/cli-v3/src/executions/errors.ts
-103
View File
@@ -1,103 +0,0 @@
import { z } from "zod";
export class UncaughtExceptionError extends Error {
constructor(
public readonly originalError: { name: string; message: string; stack?: string },
public readonly origin: "uncaughtException" | "unhandledRejection"
) {
super(`Uncaught exception: ${originalError.message}`);
this.name = "UncaughtExceptionError";
}
}
export class TaskMetadataParseError extends Error {
constructor(
public readonly zodIssues: z.ZodIssue[],
public readonly tasks: any
) {
super(`Failed to parse task metadata`);
this.name = "TaskMetadataParseError";
}
}
export class UnexpectedExitError extends Error {
constructor(
public code: number,
public signal: NodeJS.Signals | null,
public stderr: string | undefined
) {
super(`Unexpected exit with code ${code}`);
this.name = "UnexpectedExitError";
}
}
export class CleanupProcessError extends Error {
constructor() {
super("Cancelled");
this.name = "CleanupProcessError";
}
}
export class CancelledProcessError extends Error {
constructor() {
super("Cancelled");
this.name = "CancelledProcessError";
}
}
export class SigKillTimeoutProcessError extends Error {
constructor() {
super("Process kill timeout");
this.name = "SigKillTimeoutProcessError";
}
}
export class GracefulExitTimeoutError extends Error {
constructor() {
super("Graceful exit timeout");
this.name = "GracefulExitTimeoutError";
}
}
export function getFriendlyErrorMessage(
code: number,
signal: NodeJS.Signals | null,
stderr: string | undefined,
dockerMode = true
) {
const message = (text: string) => {
if (signal) {
return `[${signal}] ${text}`;
} else {
return text;
}
};
if (code === 137) {
if (dockerMode) {
return message(
"Process ran out of memory! Try choosing a machine preset with more memory for this task."
);
} else {
// Note: containerState reason and message should be checked to clarify the error
return message(
"Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task."
);
}
}
if (stderr?.includes("OOMErrorHandler")) {
return message(
"Process ran out of memory! Try choosing a machine preset with more memory for this task."
);
}
return message(`Process exited with code ${code}.`);
}
packages/cli-v3/src/executions/taskRunProcess.ts
+7 -7
View File
@@ -22,9 +22,9 @@ import {
CancelledProcessError,
CleanupProcessError,
GracefulExitTimeoutError,
SigKillTimeoutProcessError,
UnexpectedExitError,
} from "./errors.js";
} from "@trigger.dev/core/v3/errors";
import { env } from "std-env";
export type OnWaitForDurationMessage = InferSocketMessageSchema<
typeof ExecutorToWorkerMessageCatalog,
@@ -105,18 +105,18 @@ export class TaskRunProcess {
}
async initialize() {
const { env, workerManifest, cwd, messageId } = this.options;
const { env: $env, workerManifest, cwd, messageId } = this.options;
const fullEnv = {
...(this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {}),
...env,
...$env,
OTEL_IMPORT_HOOK_INCLUDES: workerManifest.otelImportHook?.include?.join(","),
// TODO: this will probably need to use something different for bun (maybe --preload?)
NODE_OPTIONS: workerManifest.loaderEntryPoint
? `--import=${workerManifest.loaderEntryPoint} ${
env.NODE_OPTIONS ?? process.env.NODE_OPTIONS ?? ""
env.NODE_OPTIONS ?? env.NODE_OPTIONS ?? ""
}`
: env.NODE_OPTIONS ?? process.env.NODE_OPTIONS ?? "",
: env.NODE_OPTIONS ?? env.NODE_OPTIONS ?? "",
};
logger.debug(`[${this.runId}] initializing task run process`, {
@@ -125,7 +125,7 @@ export class TaskRunProcess {
cwd,
});
this._child = fork(workerManifest.executorEntryPoint, executorArgs(workerManifest), {
this._child = fork(workerManifest.workerEntryPoint, executorArgs(workerManifest), {
stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
cwd,
env: fullEnv,
packages/cli-v3/src/indexing/indexWorkerManifest.ts
+118
View File
@@ -0,0 +1,118 @@
import { execPathForRuntime } from "@trigger.dev/core/v3/build";
import {
BuildRuntime,
indexerToWorkerMessages,
WorkerManifest,
} from "@trigger.dev/core/v3/schemas";
import { parseMessageFromCatalog } from "@trigger.dev/core/v3/zodMessageHandler";
import { fork } from "node:child_process";
import { env } from "std-env";
import {
TaskIndexingImportError,
TaskMetadataParseError,
UncaughtExceptionError,
} from "@trigger.dev/core/v3/errors";
export type IndexWorkerManifestOptions = {
runtime: BuildRuntime;
indexWorkerPath: string;
buildManifestPath: string;
nodeOptions?: string;
env: Record<string, string | undefined>;
cwd?: string;
otelHookInclude?: string[];
otelHookExclude?: string[];
handleStdout?: (data: string) => void;
handleStderr?: (data: string) => void;
};
export async function indexWorkerManifest({
runtime,
indexWorkerPath,
buildManifestPath,
nodeOptions,
env: $env,
cwd,
otelHookInclude,
otelHookExclude,
handleStderr,
handleStdout,
}: IndexWorkerManifestOptions) {
return await new Promise<WorkerManifest>((resolve, reject) => {
let resolved = false;
const child = fork(indexWorkerPath, {
stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
cwd,
env: {
...$env,
OTEL_IMPORT_HOOK_INCLUDES: otelHookInclude?.join(","),
OTEL_IMPORT_HOOK_EXCLUDES: otelHookExclude?.join(","),
TRIGGER_BUILD_MANIFEST_PATH: buildManifestPath,
NODE_OPTIONS: nodeOptions ? `${nodeOptions} ${env.NODE_OPTIONS ?? ""}` : env.NODE_OPTIONS,
},
execPath: execPathForRuntime(runtime),
});
// Set a timeout to kill the child process if it doesn't respond
const timeout = setTimeout(() => {
if (resolved) {
return;
}
resolved = true;
child.kill();
reject(new Error("Worker timed out"));
}, 20_000);
child.on("message", async (msg: any) => {
const message = parseMessageFromCatalog(msg, indexerToWorkerMessages);
switch (message.type) {
case "INDEX_COMPLETE": {
clearTimeout(timeout);
resolved = true;
if (message.payload.importErrors.length > 0) {
reject(
new TaskIndexingImportError(message.payload.importErrors, message.payload.manifest)
);
} else {
resolve(message.payload.manifest);
}
child.kill();
break;
}
case "TASKS_FAILED_TO_PARSE": {
clearTimeout(timeout);
resolved = true;
reject(new TaskMetadataParseError(message.payload.zodIssues, message.payload.tasks));
child.kill();
break;
}
case "UNCAUGHT_EXCEPTION": {
clearTimeout(timeout);
resolved = true;
reject(new UncaughtExceptionError(message.payload.error, message.payload.origin));
child.kill();
break;
}
}
});
child.on("exit", (code) => {
if (!resolved) {
clearTimeout(timeout);
resolved = true;
reject(new Error(`Worker exited with code ${code}`));
}
});
child.stdout?.on("data", (data) => {
handleStdout?.(data.toString());
});
child.stderr?.on("data", (data) => {
handleStderr?.(data.toString());
});
});
}
packages/cli-v3/src/indexing/registerTasks.ts
+68
View File
@@ -0,0 +1,68 @@
import { BuildManifest, ImportTaskFileErrors, taskCatalog } from "@trigger.dev/core/v3";
export async function registerTasks(buildManifest: BuildManifest): Promise<ImportTaskFileErrors> {
const importErrors: ImportTaskFileErrors = [];
for (const file of buildManifest.files) {
const [error, module] = await tryImport(file.out);
if (error) {
importErrors.push({
file: file.entry,
message: error.message,
stack: error.stack,
name: error.name,
});
continue;
}
for (const exportName of getExportNames(module)) {
const task = module[exportName] ?? module.default?.[exportName];
if (!task) {
continue;
}
if (task[Symbol.for("trigger.dev/task")]) {
if (taskCatalog.taskExists(task.id)) {
taskCatalog.registerTaskFileMetadata(task.id, {
exportName,
filePath: file.entry,
entryPoint: file.out,
});
}
}
}
}
return importErrors;
}
type Result<T> = [Error | null, T | null];
async function tryImport(path: string): Promise<Result<any>> {
try {
const module = await import(path);
return [null, module];
} catch (error) {
return [error as Error, null];
}
}
function getExportNames(module: any) {
const exports: string[] = [];
const exportKeys = Object.keys(module);
if (exportKeys.length === 0) {
return exports;
}
if (exportKeys.length === 1 && exportKeys[0] === "default") {
return Object.keys(module.default);
}
return exportKeys;
}
packages/cli-v3/src/package.json
+3
View File
@@ -0,0 +1,3 @@
{
"type": "module"
}
packages/cli-v3/src/telemetry/tracing.ts
+4 -3
View File
@@ -10,18 +10,19 @@ import {
} from "@opentelemetry/semantic-conventions";
import { logger } from "../utilities/logger.js";
import { VERSION } from "../version.js";
import { env } from "std-env";
function initializeTracing(): NodeTracerProvider | undefined {
if (
process.argv.includes("--skip-telemetry") ||
process.env.TRIGGER_DEV_SKIP_TELEMETRY || // only for backwards compat
process.env.TRIGGER_TELEMETRY_DISABLED
env.TRIGGER_DEV_SKIP_TELEMETRY || // only for backwards compat
env.TRIGGER_TELEMETRY_DISABLED
) {
logger.debug("📉 Telemetry disabled");
return;
}
if (process.env.OTEL_INTERNAL_DIAG_DEBUG) {
if (env.OTEL_INTERNAL_DIAG_DEBUG) {
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);
}
packages/cli-v3/src/utilities/cliOutput.ts
+3 -2
View File
@@ -1,8 +1,9 @@
import { log } from "@clack/prompts";
import chalk from "chalk";
import terminalLink, { Options as TerminalLinkOptions } from "terminal-link";
import { hasTTY } from "std-env";
export const isInteractive = process.stdin.isTTY;
export const isInteractive = hasTTY;
export const green = "#4FFF54";
export const purple = "#735BF3";
@@ -81,7 +82,7 @@ export function prettyError(header: string, body?: string, footer?: string) {
.map((line) => `${indent}${line}`)
.join("\n");
const prettyBody = withIndents(body);
const prettyBody = withIndents(body?.trim());
const prettyFooter = withIndents(footer);
log.error(
packages/cli-v3/src/utilities/dotEnv.ts
+2 -1
View File
@@ -1,5 +1,6 @@
import dotenv from "dotenv";
import { resolve } from "node:path";
import { env } from "std-env";
export function resolveDotEnvVars(cwd?: string) {
const result: { [key: string]: string } = {};
@@ -11,7 +12,7 @@ export function resolveDotEnvVars(cwd?: string) {
),
});
process.env.TRIGGER_API_URL && (result.TRIGGER_API_URL = process.env.TRIGGER_API_URL);
env.TRIGGER_API_URL && (result.TRIGGER_API_URL = env.TRIGGER_API_URL);
// remove TRIGGER_API_URL and TRIGGER_SECRET_KEY, since those should be coming from the worker
delete result.TRIGGER_API_URL;
packages/cli-v3/src/utilities/eventBus.ts
+7 -1
View File
@@ -1,4 +1,9 @@
import { BuildTarget, TaskRunExecutionPayload, TaskRunExecutionResult } from "@trigger.dev/core/v3";
import {
BuildManifest,
BuildTarget,
TaskRunExecutionPayload,
TaskRunExecutionResult,
} from "@trigger.dev/core/v3";
import { EventEmitter } from "node:events";
import { BackgroundWorker } from "../dev/backgroundWorker.js";
@@ -7,6 +12,7 @@ export type EventBusEvents = {
buildStarted: [BuildTarget];
workerSkipped: [];
backgroundWorkerInitialized: [BackgroundWorker];
backgroundWorkerIndexingError: [BuildManifest, Error];
runStarted: [BackgroundWorker, TaskRunExecutionPayload];
runCompleted: [BackgroundWorker, TaskRunExecutionPayload, TaskRunExecutionResult, number];
};
packages/cli-v3/src/utilities/getEnvironmentVariableFactory.ts
-69
View File
@@ -1,69 +0,0 @@
import { logger } from "./logger.js";
type VariableNames = "TRIGGER_API_URL" | "TRIGGER_SECRET_KEY" | "TRIGGER_LOG_LEVEL";
type DeprecatedNames = "";
/**
* Create a function used to access an environment variable.
*
* This is not memoized to allow us to change the value at runtime, such as in testing.
* A warning is shown if the client is using a deprecated version - but only once.
*/
export function getEnvironmentVariableFactory({
variableName,
deprecatedName,
}: {
variableName: VariableNames;
deprecatedName?: DeprecatedNames;
}): () => string | undefined;
/**
* Create a function used to access an environment variable, with a default value.
*
* This is not memoized to allow us to change the value at runtime, such as in testing.
* A warning is shown if the client is using a deprecated version - but only once.
*/
export function getEnvironmentVariableFactory({
variableName,
deprecatedName,
defaultValue,
}: {
variableName: VariableNames;
deprecatedName?: DeprecatedNames;
defaultValue: () => string;
}): () => string;
/**
* Create a function used to access an environment variable.
*
* This is not memoized to allow us to change the value at runtime, such as in testing.
* A warning is shown if the client is using a deprecated version - but only once.
*/
export function getEnvironmentVariableFactory({
variableName,
deprecatedName,
defaultValue,
}: {
variableName: VariableNames;
deprecatedName?: DeprecatedNames;
defaultValue?: () => string;
}): () => string | undefined {
let hasWarned = false;
return () => {
if (process.env[variableName]) {
return process.env[variableName];
} else if (deprecatedName && process.env[deprecatedName]) {
if (!hasWarned) {
// Only show the warning once.
hasWarned = true;
logger.warn(
`Using "${deprecatedName}" environment variable. This is deprecated. Please use "${variableName}", instead.`
);
}
return process.env[deprecatedName];
} else {
return defaultValue?.();
}
};
}
packages/cli-v3/src/utilities/logger.ts
+3 -6
View File
@@ -5,7 +5,8 @@ import chalk from "chalk";
import CLITable from "cli-table3";
import { formatMessagesSync } from "esbuild";
import type { Message } from "esbuild";
import { getEnvironmentVariableFactory } from "./getEnvironmentVariableFactory.js";
import { env } from "std-env";
export const LOGGER_LEVELS = {
none: -1,
error: 0,
@@ -26,12 +27,8 @@ const LOGGER_LEVEL_FORMAT_TYPE_MAP = {
debug: undefined,
} as const;
const getLogLevelFromEnv = getEnvironmentVariableFactory({
variableName: "TRIGGER_LOG_LEVEL",
});
function getLoggerLevel(): LoggerLevel {
const fromEnv = getLogLevelFromEnv()?.toLowerCase();
const fromEnv = env.TRIGGER_LOG_LEVEL?.toLowerCase();
if (fromEnv !== undefined) {
if (fromEnv in LOGGER_LEVELS) return fromEnv as LoggerLevel;
const expected = Object.keys(LOGGER_LEVELS)
packages/cli-v3/src/utilities/windows.ts
+2 -1
View File
@@ -1,6 +1,7 @@
import { log, spinner as clackSpinner } from "@clack/prompts";
import { isWindows as stdEnvIsWindows } from "std-env";
export const isWindows = process.platform === "win32";
export const isWindows = stdEnvIsWindows;
export function escapeImportPath(path: string) {
return isWindows ? path.replaceAll("\\", "\\\\") : path;
packages/core/src/v3/errors.ts
+275 -11
View File
@@ -1,5 +1,8 @@
import { z } from "zod";
import { TaskRunError } from "./schemas/common.js";
import { DeploymentErrorData } from "./schemas/api.js";
import { ImportTaskFileErrors, WorkerManifest } from "./schemas/build.js";
import { SerializedError, TaskRunError } from "./schemas/common.js";
import { TaskMetadataFailedToParseData } from "./schemas/messages.js";
export class AbortTaskRunError extends Error {
constructor(message: string) {
@@ -60,14 +63,6 @@ export function createErrorTaskError(error: TaskRunError): any {
}
}
export const SerializedError = z.object({
message: z.string(),
name: z.string().optional(),
stackTrace: z.string().optional(),
});
export type SerializedError = z.infer<typeof SerializedError>;
export function createJsonErrorObject(error: TaskRunError): SerializedError {
switch (error.type) {
case "BUILT_IN_ERROR": {
@@ -178,7 +173,7 @@ export function groupTaskMetadataIssuesByTask(tasks: any, issues: z.ZodIssue[])
return acc;
}
const taskIndex = issue.path[1];
const taskIndex = issue.path[2];
if (typeof taskIndex !== "number") {
return acc;
@@ -190,7 +185,7 @@ export function groupTaskMetadataIssuesByTask(tasks: any, issues: z.ZodIssue[])
return acc;
}
const restOfPath = issue.path.slice(2);
const restOfPath = issue.path.slice(3);
const taskId = task.id;
const taskName = task.exportName;
@@ -226,3 +221,272 @@ export function groupTaskMetadataIssuesByTask(tasks: any, issues: z.ZodIssue[])
>
);
}
export class UncaughtExceptionError extends Error {
constructor(
public readonly originalError: { name: string; message: string; stack?: string },
public readonly origin: "uncaughtException" | "unhandledRejection"
) {
super(`Uncaught exception: ${originalError.message}`);
this.name = "UncaughtExceptionError";
}
}
export class TaskMetadataParseError extends Error {
constructor(
public readonly zodIssues: z.ZodIssue[],
public readonly tasks: any
) {
super(`Failed to parse task metadata`);
this.name = "TaskMetadataParseError";
}
}
export class TaskIndexingImportError extends Error {
constructor(
public readonly importErrors: ImportTaskFileErrors,
public readonly manifest: WorkerManifest
) {
super(`Failed to import some task files`);
this.name = "TaskIndexingImportError";
}
}
export class UnexpectedExitError extends Error {
constructor(
public code: number,
public signal: NodeJS.Signals | null,
public stderr: string | undefined
) {
super(`Unexpected exit with code ${code}`);
this.name = "UnexpectedExitError";
}
}
export class CleanupProcessError extends Error {
constructor() {
super("Cancelled");
this.name = "CleanupProcessError";
}
}
export class CancelledProcessError extends Error {
constructor() {
super("Cancelled");
this.name = "CancelledProcessError";
}
}
export class SigKillTimeoutProcessError extends Error {
constructor() {
super("Process kill timeout");
this.name = "SigKillTimeoutProcessError";
}
}
export class GracefulExitTimeoutError extends Error {
constructor() {
super("Graceful exit timeout");
this.name = "GracefulExitTimeoutError";
}
}
export function getFriendlyErrorMessage(
code: number,
signal: NodeJS.Signals | null,
stderr: string | undefined,
dockerMode = true
) {
const message = (text: string) => {
if (signal) {
return `[${signal}] ${text}`;
} else {
return text;
}
};
if (code === 137) {
if (dockerMode) {
return message(
"Process ran out of memory! Try choosing a machine preset with more memory for this task."
);
} else {
// Note: containerState reason and message should be checked to clarify the error
return message(
"Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task."
);
}
}
if (stderr?.includes("OOMErrorHandler")) {
return message(
"Process ran out of memory! Try choosing a machine preset with more memory for this task."
);
}
return message(`Process exited with code ${code}.`);
}
export function serializeIndexingError(error: unknown, stderr?: string): DeploymentErrorData {
if (error instanceof TaskMetadataParseError) {
return {
name: "TaskMetadataParseError",
message: "There was an error parsing the task metadata",
stack: JSON.stringify({ zodIssues: error.zodIssues, tasks: error.tasks }),
stderr,
};
} else if (error instanceof TaskIndexingImportError) {
return {
name: "TaskIndexingImportError",
message: "There was an error importing task files",
stack: JSON.stringify(error.importErrors),
stderr,
};
} else if (error instanceof UncaughtExceptionError) {
const originalError = error.originalError;
return {
name: originalError.name,
message: originalError.message,
stack: originalError.stack,
stderr,
};
} else if (error instanceof Error) {
return {
name: error.name,
message: error.message,
stack: error.stack,
stderr,
};
}
return {
name: "UnknownError",
message: String(error),
stderr,
};
}
export function prepareDeploymentError(
errorData: DeploymentErrorData
): DeploymentErrorData | undefined {
if (!errorData) {
return;
}
if (errorData.name === "TaskMetadataParseError") {
const errorJson = tryJsonParse(errorData.stack);
if (errorJson) {
const parsedError = TaskMetadataFailedToParseData.safeParse(errorJson);
if (parsedError.success) {
return {
name: errorData.name,
message: errorData.message,
stack: createTaskMetadataFailedErrorStack(parsedError.data),
stderr: errorData.stderr,
};
} else {
return {
name: errorData.name,
message: errorData.message,
stderr: errorData.stderr,
};
}
} else {
return {
name: errorData.name,
message: errorData.message,
stderr: errorData.stderr,
};
}
} else if (errorData.name === "TaskIndexingImportError") {
const errorJson = tryJsonParse(errorData.stack);
if (errorJson) {
const parsedError = ImportTaskFileErrors.safeParse(errorJson);
if (parsedError.success) {
return {
name: errorData.name,
message: errorData.message,
stack: parsedError.data
.map((error) => {
return `x ${error.message} in ${error.file}`;
})
.join("\n"),
stderr: errorData.stderr,
};
} else {
return {
name: errorData.name,
message: errorData.message,
stderr: errorData.stderr,
};
}
} else {
return {
name: errorData.name,
message: errorData.message,
stderr: errorData.stderr,
};
}
}
return {
name: errorData.name,
message: errorData.message,
stack: errorData.stack,
stderr: errorData.stderr,
};
}
export function createTaskMetadataFailedErrorStack(
data: z.infer<typeof TaskMetadataFailedToParseData>
): string {
const stack = [];
const groupedIssues = groupTaskMetadataIssuesByTask(data.tasks, data.zodIssues);
for (const key in groupedIssues) {
const taskWithIssues = groupedIssues[key];
if (!taskWithIssues) {
continue;
}
stack.push("\n");
stack.push(` ❯ ${taskWithIssues.exportName} in ${taskWithIssues.filePath}`);
for (const issue of taskWithIssues.issues) {
if (issue.path) {
stack.push(` x ${issue.path} ${issue.message}`);
} else {
stack.push(` x ${issue.message}`);
}
}
}
return stack.join("\n");
}
function tryJsonParse(data: string | undefined): any {
if (!data) {
return;
}
try {
return JSON.parse(data);
} catch {
return;
}
}
packages/core/src/v3/schemas/api.ts
+4 -2
View File
@@ -1,7 +1,7 @@
import { z } from "zod";
import { BackgroundWorkerMetadata, ImageDetailsMetadata } from "./resources.js";
import { BackgroundWorkerMetadata } from "./resources.js";
import { QueueOptions } from "./schemas.js";
import { SerializedError } from "../errors.js";
import { SerializedError } from "./common.js";
export const WhoAmIResponseSchema = z.object({
userId: z.string(),
@@ -195,6 +195,8 @@ export const DeploymentErrorData = z.object({
stderr: z.string().optional(),
});
export type DeploymentErrorData = z.infer<typeof DeploymentErrorData>;
export const FailDeploymentRequestBody = z.object({
error: DeploymentErrorData,
});
packages/core/src/v3/schemas/build.ts
+19 -5
View File
@@ -33,9 +33,10 @@ export const BuildManifest = z.object({
})
),
outputPath: z.string(),
indexerEntryPoint: z.string(),
executorEntryPoint: z.string(),
workerEntryPoint: z.string().optional(),
runWorkerEntryPoint: z.string(), // Dev & Deploy has a runWorkerEntryPoint
runControllerEntryPoint: z.string().optional(), // Only deploy has a runControllerEntryPoint
indexWorkerEntryPoint: z.string(), // Dev & Deploy has a indexWorkerEntryPoint
indexControllerEntryPoint: z.string().optional(), // Only deploy has a indexControllerEntryPoint
loaderEntryPoint: z.string().optional(),
configPath: z.string(),
externals: BuildExternal.array().optional(),
@@ -73,8 +74,8 @@ export type IndexMessage = z.infer<typeof IndexMessage>;
export const WorkerManifest = z.object({
configPath: z.string(),
tasks: TaskManifest.array(),
executorEntryPoint: z.string(),
workerEntryPoint: z.string().optional(),
workerEntryPoint: z.string(),
controllerEntryPoint: z.string().optional(),
loaderEntryPoint: z.string().optional(),
runtime: BuildRuntime,
otelImportHook: z
@@ -95,3 +96,16 @@ export const WorkerManifestMessage = z.object({
});
export type WorkerManifestMessage = z.infer<typeof WorkerManifestMessage>;
export const ImportError = z.object({
message: z.string(),
file: z.string(),
stack: z.string().optional(),
name: z.string().optional(),
});
export type ImportError = z.infer<typeof ImportError>;
export const ImportTaskFileErrors = z.array(ImportError);
export type ImportTaskFileErrors = z.infer<typeof ImportTaskFileErrors>;
packages/core/src/v3/schemas/common.ts
+8
View File
@@ -280,3 +280,11 @@ export const BatchTaskRunExecutionResult = z.object({
});
export type BatchTaskRunExecutionResult = z.infer<typeof BatchTaskRunExecutionResult>;
export const SerializedError = z.object({
message: z.string(),
name: z.string().optional(),
stackTrace: z.string().optional(),
});
export type SerializedError = z.infer<typeof SerializedError>;
packages/core/src/v3/schemas/messages.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import { z } from "zod";
import { WorkerManifest } from "./build.js";
import { ImportTaskFileErrors, WorkerManifest } from "./build.js";
import {
MachinePreset,
TaskRunExecution,
@@ -11,7 +11,6 @@ import {
EnvironmentType,
ProdTaskRunExecution,
ProdTaskRunExecutionPayload,
TaskManifest,
TaskRunExecutionLazyAttemptPayload,
WaitReason,
} from "./schemas.js";
@@ -129,6 +128,7 @@ export const indexerToWorkerMessages = {
INDEX_COMPLETE: z.object({
version: z.literal("v1").default("v1"),
manifest: WorkerManifest,
importErrors: ImportTaskFileErrors,
}),
TASKS_FAILED_TO_PARSE: TaskMetadataFailedToParseData,
UNCAUGHT_EXCEPTION: UncaughtExceptionMessage,
pnpm-lock.yaml
Generated
+20 -6
View File
@@ -978,6 +978,9 @@ importers:
source-map-support:
specifier: 0.5.21
version: 0.5.21
std-env:
specifier: ^3.7.0
version: 3.7.0
terminal-link:
specifier: ^3.0.0
version: 3.0.0
@@ -1360,6 +1363,9 @@ importers:
'@sindresorhus/slugify':
specifier: ^2.2.1
version: 2.2.1
'@t3-oss/env-core':
specifier: ^0.11.0
version: 0.11.0(typescript@5.5.4)(zod@3.23.8)
'@t3-oss/env-nextjs':
specifier: ^0.10.1
version: 0.10.1(typescript@5.5.4)(zod@3.23.8)
@@ -12287,6 +12293,19 @@ packages:
zod: 3.23.8
dev: false
/@t3-oss/env-core@0.11.0(typescript@5.5.4)(zod@3.23.8):
resolution: {integrity: sha512-PSalC5bG0a7XbyoLydiQdAnx3gICX6IQNctvh+TyLrdFxsxgocdj9Ui7sd061UlBzi+z4aIGjnem1kZx9QtUgQ==}
peerDependencies:
typescript: '>=5.0.0'
zod: ^3.0.0
peerDependenciesMeta:
typescript:
optional: true
dependencies:
typescript: 5.5.4
zod: 3.23.8
dev: false
/@t3-oss/env-nextjs@0.10.1(typescript@5.5.4)(zod@3.23.8):
resolution: {integrity: sha512-iy2qqJLnFh1RjEWno2ZeyTu0ufomkXruUsOZludzDIroUabVvHsrSjtkHqwHp1/pgPUzN3yBRHMILW162X7x2Q==}
peerDependencies:
@@ -25489,13 +25508,8 @@ packages:
resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
engines: {node: '>= 0.8'}
/std-env@3.3.2:
resolution: {integrity: sha512-uUZI65yrV2Qva5gqE0+A7uVAvO40iPo6jGhs7s8keRfHCmtg+uB2X6EiLGCI9IgL1J17xGhvoOqSz79lzICPTA==}
dev: true
/std-env@3.7.0:
resolution: {integrity: sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==}
dev: true
/stoppable@1.1.0:
resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==}
@@ -27761,7 +27775,7 @@ packages:
pathe: 1.1.0
picocolors: 1.0.0
source-map: 0.6.1
std-env: 3.3.2
std-env: 3.7.0
strip-literal: 1.0.1
tinybench: 2.3.1
tinypool: 0.3.1
references/v3-catalog/package.json
+1
View File
@@ -23,6 +23,7 @@
"@react-email/components": "^0.0.17",
"@react-email/render": "^0.0.7",
"@sindresorhus/slugify": "^2.2.1",
"@t3-oss/env-core": "^0.11.0",
"@t3-oss/env-nextjs": "^0.10.1",
"@traceloop/instrumentation-openai": "^0.10.0",
"@trigger.dev/sdk": "workspace:*",
references/v3-catalog/src/env.ts
+13
View File
@@ -0,0 +1,13 @@
import { createEnv } from "@t3-oss/env-core";
import { z } from "zod";
export const env = createEnv({
server: {
UPLOADTHING_SECRET: z.string(),
UPLOADTHING_APP_ID: z.string(),
OPENAI_API_KEY: z.string(),
},
runtimeEnv: process.env,
emptyStringAsUndefined: true,
});
references/v3-catalog/src/trigger/openai.mts
+2 -1
View File
@@ -1,9 +1,10 @@
import { env } from "@/env.js";
import { logger, task } from "@trigger.dev/sdk/v3";
import OpenAI from "openai";
const openai = new OpenAI({
apiKey: process.env.OPENAI_API_KEY,
apiKey: env.OPENAI_API_KEY,
});
export const openaiTask = task({