deployment WIP

This commit is contained in:
Eric Allam
2024-08-17 19:36:13 +01:00
committed by Eric Allam
parent 104301a104
commit 059f1887f7
50 changed files with 2669 additions and 1471 deletions
+16
View File
@@ -45,6 +45,22 @@
"cwd": "${workspaceFolder}/references/v3-catalog",
"sourceMaps": true
},
{
"type": "node-terminal",
"request": "launch",
"name": "Debug V3 list-profiles CLI",
"command": "pnpm exec triggerdev list-profiles --log-level debug",
"cwd": "${workspaceFolder}/references/v3-catalog",
"sourceMaps": true
},
{
"type": "node-terminal",
"request": "launch",
"name": "Debug V3 update CLI",
"command": "pnpm exec triggerdev update",
"cwd": "${workspaceFolder}/references/v3-catalog",
"sourceMaps": true
},
{
"type": "node-terminal",
"request": "launch",
@@ -0,0 +1,73 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
import { z } from "zod";
import { authenticateApiRequest } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { ServiceValidationError } from "~/v3/services/baseService.server";
import { CreateDeclarativeScheduleError } from "~/v3/services/createBackgroundWorker.server";
import { CreateDeploymentBackgroundWorkerService } from "~/v3/services/createDeploymentBackgroundWorker.server";
const ParamsSchema = z.object({
deploymentId: z.string(),
});
export async function action({ request, params }: ActionFunctionArgs) {
// Ensure this is a POST request
if (request.method.toUpperCase() !== "POST") {
return { status: 405, body: "Method Not Allowed" };
}
const parsedParams = ParamsSchema.safeParse(params);
if (!parsedParams.success) {
return json({ error: "Invalid params" }, { status: 400 });
}
// Next authenticate the request
const authenticationResult = await authenticateApiRequest(request);
if (!authenticationResult) {
logger.info("Invalid or missing api key", { url: request.url });
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
const authenticatedEnv = authenticationResult.environment;
const { deploymentId } = parsedParams.data;
const rawBody = await request.json();
const body = CreateBackgroundWorkerRequestBody.safeParse(rawBody);
if (!body.success) {
return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
}
const service = new CreateDeploymentBackgroundWorkerService();
try {
const backgroundWorker = await service.call(authenticatedEnv, deploymentId, body.data);
if (!backgroundWorker) {
return json({ error: "Failed to create background worker" }, { status: 500 });
}
return json(
{
id: backgroundWorker.friendlyId,
version: backgroundWorker.version,
contentHash: backgroundWorker.contentHash,
},
{ status: 200 }
);
} catch (e) {
logger.error("Failed to create background worker", { error: e });
if (e instanceof ServiceValidationError) {
return json({ error: e.message }, { status: 400 });
} else if (e instanceof CreateDeclarativeScheduleError) {
return json({ error: e.message }, { status: 400 });
}
return json({ error: "Failed to create background worker" }, { status: 500 });
}
}
@@ -0,0 +1,52 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { FailDeploymentRequestBody } from "@trigger.dev/core/v3";
import { z } from "zod";
import { authenticateApiRequest } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { FailDeploymentService } from "~/v3/services/failDeployment.server";
const ParamsSchema = z.object({
deploymentId: z.string(),
});
export async function action({ request, params }: ActionFunctionArgs) {
// Ensure this is a POST request
if (request.method.toUpperCase() !== "POST") {
return { status: 405, body: "Method Not Allowed" };
}
const parsedParams = ParamsSchema.safeParse(params);
if (!parsedParams.success) {
return json({ error: "Invalid params" }, { status: 400 });
}
// Next authenticate the request
const authenticationResult = await authenticateApiRequest(request);
if (!authenticationResult) {
logger.info("Invalid or missing api key", { url: request.url });
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
const authenticatedEnv = authenticationResult.environment;
const { deploymentId } = parsedParams.data;
const rawBody = await request.json();
const body = FailDeploymentRequestBody.safeParse(rawBody);
if (!body.success) {
return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
}
const service = new FailDeploymentService();
await service.call(authenticatedEnv, deploymentId, body.data);
return json(
{
id: deploymentId,
},
{ status: 200 }
);
}
@@ -0,0 +1,65 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3";
import { z } from "zod";
import { authenticateApiRequest } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { ServiceValidationError } from "~/v3/services/baseService.server";
import { FinalizeDeploymentService } from "~/v3/services/finalizeDeployment.server";
const ParamsSchema = z.object({
deploymentId: z.string(),
});
export async function action({ request, params }: ActionFunctionArgs) {
// Ensure this is a POST request
if (request.method.toUpperCase() !== "POST") {
return { status: 405, body: "Method Not Allowed" };
}
const parsedParams = ParamsSchema.safeParse(params);
if (!parsedParams.success) {
return json({ error: "Invalid params" }, { status: 400 });
}
// Next authenticate the request
const authenticationResult = await authenticateApiRequest(request);
if (!authenticationResult) {
logger.info("Invalid or missing api key", { url: request.url });
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
const authenticatedEnv = authenticationResult.environment;
const { deploymentId } = parsedParams.data;
const rawBody = await request.json();
const body = FinalizeDeploymentRequestBody.safeParse(rawBody);
if (!body.success) {
return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
}
try {
const service = new FinalizeDeploymentService();
await service.call(authenticatedEnv, deploymentId, body.data);
return json(
{
id: deploymentId,
},
{ status: 200 }
);
} catch (error) {
if (error instanceof ServiceValidationError) {
return json({ error: error.message }, { status: 400 });
} else if (error instanceof Error) {
logger.error("Error finalizing deployment", { error: error.message });
return json({ error: `Internal server error: ${error.message}` }, { status: 500 });
} else {
logger.error("Error finalizing deployment", { error: String(error) });
return json({ error: "Internal server error" }, { status: 500 });
}
}
}
@@ -89,6 +89,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
apiKey: runtimeEnv.apiKey,
name: project.name,
apiUrl: processEnv.APP_ORIGIN,
projectId: project.id,
};
return json(result);
@@ -1,5 +1,5 @@
import {
BackgroundWorkerFileMetadata,
BackgroundWorkerSourceFileMetadata,
CreateBackgroundWorkerRequestBody,
TaskResource,
} from "@trigger.dev/core/v3";
@@ -75,7 +75,7 @@ export class CreateBackgroundWorkerService extends BaseService {
});
const tasksToBackgroundFiles = await createBackgroundFiles(
body.metadata.fileContents,
body.metadata.sourceFiles,
backgroundWorker,
environment,
this._prisma
@@ -401,7 +401,7 @@ export async function syncDeclarativeSchedules(
}
export async function createBackgroundFiles(
files: Array<BackgroundWorkerFileMetadata> | undefined,
files: Array<BackgroundWorkerSourceFileMetadata> | undefined,
worker: BackgroundWorker,
environment: AuthenticatedEnvironment,
prisma: PrismaClientOrTransaction
@@ -0,0 +1,108 @@
import { CreateBackgroundWorkerRequestBody } from "@trigger.dev/core/v3";
import type { BackgroundWorker } from "@trigger.dev/database";
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { generateFriendlyId } from "../friendlyIdentifiers";
import { BaseService } from "./baseService.server";
import {
createBackgroundFiles,
createBackgroundTasks,
syncDeclarativeSchedules,
} from "./createBackgroundWorker.server";
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
export class CreateDeploymentBackgroundWorkerService extends BaseService {
public async call(
environment: AuthenticatedEnvironment,
deploymentId: string,
body: CreateBackgroundWorkerRequestBody
): Promise<BackgroundWorker | undefined> {
return this.traceWithEnv("call", environment, async (span) => {
span.setAttribute("deploymentId", deploymentId);
const deployment = await this._prisma.workerDeployment.findUnique({
where: {
friendlyId: deploymentId,
},
});
if (!deployment) {
return;
}
if (deployment.status !== "BUILDING") {
return;
}
const backgroundWorker = await this._prisma.backgroundWorker.create({
data: {
friendlyId: generateFriendlyId("worker"),
version: deployment.version,
runtimeEnvironmentId: environment.id,
projectId: environment.projectId,
metadata: body.metadata,
contentHash: body.metadata.contentHash,
cliVersion: body.metadata.cliPackageVersion,
sdkVersion: body.metadata.packageVersion,
supportsLazyAttempts: body.supportsLazyAttempts,
},
});
try {
const tasksToBackgroundFiles = await createBackgroundFiles(
body.metadata.sourceFiles,
backgroundWorker,
environment,
this._prisma
);
await createBackgroundTasks(
body.metadata.tasks,
backgroundWorker,
environment,
this._prisma,
tasksToBackgroundFiles
);
await syncDeclarativeSchedules(
body.metadata.tasks,
backgroundWorker,
environment,
this._prisma
);
} catch (error) {
const name = error instanceof Error ? error.name : "UnknownError";
const message = error instanceof Error ? error.message : JSON.stringify(error);
await this._prisma.workerDeployment.update({
where: {
id: deployment.id,
},
data: {
status: "FAILED",
failedAt: new Date(),
errorData: {
name,
message,
},
},
});
throw error;
}
// Link the deployment with the background worker
await this._prisma.workerDeployment.update({
where: {
id: deployment.id,
},
data: {
status: "DEPLOYING",
workerId: backgroundWorker.id,
deployedAt: new Date(),
},
});
await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
return backgroundWorker;
});
}
}
@@ -0,0 +1,56 @@
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
import { BaseService } from "./baseService.server";
import { logger } from "~/services/logger.server";
import { WorkerDeploymentStatus } from "@trigger.dev/database";
import { FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
"CANCELED",
"DEPLOYED",
"FAILED",
"TIMED_OUT",
];
export class FailDeploymentService extends BaseService {
public async call(
authenticatedEnv: AuthenticatedEnvironment,
id: string,
params: FailDeploymentRequestBody
) {
const deployment = await this._prisma.workerDeployment.findUnique({
where: {
friendlyId: id,
environmentId: authenticatedEnv.id,
},
});
if (!deployment) {
logger.error("Worker deployment not found", { id });
return;
}
if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
logger.error("Worker deployment already in final state", {
id: deployment.id,
status: deployment.status,
});
return;
}
const failedDeployment = await this._prisma.workerDeployment.update({
where: {
id: deployment.id,
},
data: {
status: "FAILED",
failedAt: new Date(),
errorData: params.error,
},
});
await PerformDeploymentAlertsService.enqueue(failedDeployment.id, this._prisma);
return failedDeployment;
}
}
@@ -0,0 +1,122 @@
import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
import { CURRENT_DEPLOYMENT_LABEL } from "~/consts";
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { socketIo } from "../handleSocketIo.server";
import { marqs } from "../marqs/index.server";
import { registryProxy } from "../registryProxy.server";
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
import { BaseService, ServiceValidationError } from "./baseService.server";
import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy";
import { projectPubSub } from "./projectPubSub.server";
export class FinalizeDeploymentService extends BaseService {
public async call(
authenticatedEnv: AuthenticatedEnvironment,
id: string,
body: FinalizeDeploymentRequestBody
) {
const deployment = await this._prisma.workerDeployment.findUnique({
where: {
friendlyId: id,
environmentId: authenticatedEnv.id,
},
include: {
worker: {
include: {
tasks: true,
},
},
},
});
if (!deployment) {
logger.error("Worker deployment not found", { id });
return;
}
if (!deployment.worker) {
logger.error("Worker deployment does not have a worker", { id });
// TODO: We need to fail the deployment here because it's not possible to deploy a worker without a worker
throw new ServiceValidationError("Worker deployment does not have a worker");
}
if (deployment.status !== "DEPLOYING") {
logger.error("Worker deployment is not in DEPLOYING status", { id });
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
}
// Link the deployment with the background worker
const finalizedDeployment = await this._prisma.workerDeployment.update({
where: {
id: deployment.id,
},
data: {
status: "DEPLOYED",
deployedAt: new Date(),
imageReference:
registryProxy && body.selfHosted !== true
? registryProxy.rewriteImageReference(body.imageReference)
: body.imageReference,
},
});
//set this deployment as the current deployment for this environment
await this._prisma.workerDeploymentPromotion.upsert({
where: {
environmentId_label: {
environmentId: authenticatedEnv.id,
label: CURRENT_DEPLOYMENT_LABEL,
},
},
create: {
deploymentId: finalizedDeployment.id,
environmentId: authenticatedEnv.id,
label: CURRENT_DEPLOYMENT_LABEL,
},
update: {
deploymentId: finalizedDeployment.id,
},
});
try {
//send a notification that a new worker has been created
await projectPubSub.publish(
`project:${authenticatedEnv.projectId}:env:${authenticatedEnv.id}`,
"WORKER_CREATED",
{
environmentId: authenticatedEnv.id,
environmentType: authenticatedEnv.type,
createdAt: authenticatedEnv.createdAt,
taskCount: deployment.worker.tasks.length,
type: "deployed",
}
);
await marqs?.updateEnvConcurrencyLimits(authenticatedEnv);
} catch (err) {
logger.error("Failed to publish WORKER_CREATED event", { err });
}
if (deployment.imageReference) {
socketIo.providerNamespace.emit("PRE_PULL_DEPLOYMENT", {
version: "v1",
imageRef: deployment.imageReference,
shortCode: deployment.shortCode,
// identifiers
deploymentId: deployment.id,
envId: authenticatedEnv.id,
envType: authenticatedEnv.type,
orgId: authenticatedEnv.organizationId,
projectId: deployment.projectId,
});
}
await ExecuteTasksWaitingForDeployService.enqueue(deployment.worker.id, this._prisma);
await PerformDeploymentAlertsService.enqueue(deployment.id, this._prisma);
return finalizedDeployment;
}
}
+8 -33
View File
@@ -46,27 +46,21 @@
},
"devDependencies": {
"@types/gradient-string": "^1.1.2",
"@types/mock-fs": "^4.13.1",
"@types/node": "20.14.14",
"@types/object-hash": "^3.0.6",
"@types/object-hash": "3.0.6",
"@types/react": "^18.2.48",
"@types/resolve": "^1.20.6",
"@types/semver": "^7.3.13",
"@types/source-map-support": "0.5.10",
"@types/ws": "^8.5.3",
"cpy-cli": "^5.0.0",
"nodemon": "^3.0.1",
"npm-run-all": "^4.1.5",
"open": "^10.0.3",
"p-retry": "^6.1.0",
"rimraf": "^5.0.7",
"ts-essentials": "10.0.1",
"tshy": "^3.0.2",
"tsx": "4.17.0",
"type-fest": "^3.6.0",
"typescript": "^5.5.4",
"vitest": "^1.6.0",
"xdg-app-paths": "^8.3.0"
"vitest": "^1.6.0"
},
"scripts": {
"typecheck": "tsc",
@@ -78,9 +72,8 @@
"update-version": "tsx ../../scripts/updateVersion.ts"
},
"dependencies": {
"@anatine/esbuild-decorators": "^0.2.19",
"@clack/prompts": "^0.7.0",
"@depot/cli": "0.0.1-cli.2.71.0",
"@depot/cli": "0.0.1-cli.2.73.0",
"@opentelemetry/api": "1.9.0",
"@opentelemetry/api-logs": "0.52.1",
"@opentelemetry/exporter-logs-otlp-http": "0.52.1",
@@ -94,56 +87,38 @@
"@opentelemetry/sdk-trace-node": "1.25.1",
"@opentelemetry/semantic-conventions": "1.25.1",
"@trigger.dev/core": "workspace:3.0.0-beta.52",
"@types/degit": "^2.8.3",
"async-sema": "^3.1.1",
"c12": "^1.11.1",
"chalk": "^5.2.0",
"chokidar": "^3.5.3",
"cli-table3": "^0.6.3",
"commander": "^9.4.1",
"defu": "^6.1.4",
"degit": "^2.8.4",
"dotenv": "^16.4.5",
"esbuild": "^0.23.0",
"evt": "^2.4.13",
"execa": "^9.1.0",
"find-up": "^7.0.0",
"glob": "^11.0.0",
"glob-to-regexp": "^0.4.1",
"fast-npm-meta": "^0.2.2",
"gradient-string": "^2.0.2",
"hono": "^4.4.13",
"import-in-the-middle": "1.9.1",
"import-meta-resolve": "^4.1.0",
"ink": "^4.4.1",
"jsonc-parser": "3.2.1",
"liquidjs": "^10.9.2",
"magicast": "^0.3.4",
"minimatch": "^10.0.1",
"mlly": "^1.7.1",
"mock-fs": "^5.2.0",
"nanoid": "^4.0.2",
"node-fetch": "^3.3.0",
"nypm": "^0.3.9",
"object-hash": "^3.0.0",
"p-debounce": "^4.0.0",
"p-throttle": "^6.1.0",
"open": "^10.0.3",
"partysocket": "^0.0.17",
"pkg-types": "^1.1.3",
"proxy-agent": "^6.3.0",
"react": "^18.2.0",
"react-error-boundary": "^3.1.4",
"resolve": "^1.22.8",
"semver": "^7.5.0",
"signal-exit": "^4.1.0",
"simple-git": "^3.19.0",
"source-map-support": "0.5.21",
"terminal-link": "^3.0.0",
"tiny-invariant": "^1.2.0",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.4.0",
"unplugin": "^1.12.0",
"update-check": "^1.5.4",
"url": "^0.11.1",
"tinyexec": "^0.1.4",
"ws": "^8.12.0",
"xdg-app-paths": "^8.3.0",
"zod": "3.23.8",
"zod-validation-error": "^1.5.0"
},
+65 -2
View File
@@ -17,6 +17,9 @@ import {
ImportEnvironmentVariablesRequestBody,
EnvironmentVariableResponseBody,
TaskRunExecution,
FailDeploymentRequestBody,
FailDeploymentResponseBody,
FinalizeDeploymentRequestBody,
} from "@trigger.dev/core/v3";
import { zodfetch, ApiError } from "@trigger.dev/core/v3/zodfetch";
@@ -155,7 +158,7 @@ export class CliApiClient {
async importEnvVars(
projectRef: string,
slug: "dev" | "prod" | "staging",
slug: string,
params: ImportEnvironmentVariablesRequestBody
) {
if (!this.accessToken) {
@@ -191,6 +194,66 @@ export class CliApiClient {
});
}
async createDeploymentBackgroundWorker(
deploymentId: string,
body: CreateBackgroundWorkerRequestBody
) {
if (!this.accessToken) {
throw new Error("createDeploymentBackgroundWorker: No access token");
}
return wrapZodFetch(
CreateBackgroundWorkerResponse,
`${this.apiURL}/api/v1/deployments/${deploymentId}/background-workers`,
{
method: "POST",
headers: {
Authorization: `Bearer ${this.accessToken}`,
"Content-Type": "application/json",
},
body: JSON.stringify(body),
}
);
}
async failDeployment(id: string, body: FailDeploymentRequestBody) {
if (!this.accessToken) {
throw new Error("failDeployment: No access token");
}
return wrapZodFetch(
FailDeploymentResponseBody,
`${this.apiURL}/api/v1/deployments/${id}/fail`,
{
method: "POST",
headers: {
Authorization: `Bearer ${this.accessToken}`,
"Content-Type": "application/json",
},
body: JSON.stringify(body),
}
);
}
async finalizeDeployment(id: string, body: FinalizeDeploymentRequestBody) {
if (!this.accessToken) {
throw new Error("finalizeDeployment: No access token");
}
return wrapZodFetch(
FailDeploymentResponseBody,
`${this.apiURL}/api/v1/deployments/${id}/finalize`,
{
method: "POST",
headers: {
Authorization: `Bearer ${this.accessToken}`,
"Content-Type": "application/json",
},
body: JSON.stringify(body),
}
);
}
async startDeploymentIndexing(deploymentId: string, body: StartDeploymentIndexingRequestBody) {
if (!this.accessToken) {
throw new Error("startDeploymentIndexing: No access token");
@@ -245,7 +308,7 @@ async function wrapZodFetch<T extends z.ZodTypeAny>(
retry: {
minTimeoutInMs: 500,
maxTimeoutInMs: 5000,
maxAttempts: 3,
maxAttempts: 5,
factor: 2,
randomize: false,
},
+11 -2
View File
@@ -9,6 +9,7 @@ import {
deployEntryPoints,
devEntryPoints,
isDeployEntryPoint,
isDeployIndexerEntryPoint,
isDevEntryPoint,
isIndexerEntryPoint,
isLoaderEntryPoint,
@@ -72,7 +73,7 @@ export async function bundleWorker(options: BundleOptions): Promise<BundleResult
platform: "node",
sourcemap: true,
sourcesContent: options.target === "dev",
conditions: ["taskhero", "node"],
conditions: ["trigger.dev", "node"],
format: "esm",
target: ["node20", "es2022"],
loader: {
@@ -160,7 +161,7 @@ export async function getBundleResultFromBuild(
loaderEntryPoint = $outputPath;
} else if (isEntryPointForTarget(outputMeta.entryPoint, target)) {
workerEntryPoint = $outputPath;
} else if (isIndexerEntryPoint(outputMeta.entryPoint)) {
} else if (isIndexerEntryPointForTarget(outputMeta.entryPoint, target)) {
indexerEntryPoint = $outputPath;
} else {
if (
@@ -202,6 +203,14 @@ function isConfigEntryPoint(entryPoint: string) {
return entryPoint.startsWith("trigger.config.ts");
}
function isIndexerEntryPointForTarget(entryPoint: string, target: BuildTarget) {
if (target === "dev") {
return isIndexerEntryPoint(entryPoint);
} else {
return isDeployIndexerEntryPoint(entryPoint);
}
}
async function getEntryPoints(target: BuildTarget, config: ResolvedConfig) {
const projectEntryPoints = config.dirs.flatMap((dir) => dirToEntryPointGlob(dir));
+23 -1
View File
@@ -94,6 +94,11 @@ export function createBuildContext(
prependExtension(extension) {
extensions.unshift(extension);
},
logger: {
debug: (...args) => logger.debug(...args),
log: (...args) => logger.log(...args),
warn: (...args) => logger.warn(...args),
},
};
}
@@ -125,7 +130,24 @@ function applyLayerToManifest(layer: BuildLayer, manifest: BuildManifest): Build
if (layer.deploy?.env) {
manifest.deploy.env ??= {};
Object.assign(manifest.deploy.env, layer.deploy.env);
for (const [key, value] of Object.entries(layer.deploy.env)) {
if (!value) {
continue;
}
if (layer.deploy.override || manifest.deploy.env[key] === undefined) {
let needsSyncing = manifest.deploy.needsSyncing;
const existingValue = manifest.deploy.env[key];
if (existingValue !== value) {
needsSyncing = true;
}
manifest.deploy.env[key] = value;
manifest.deploy.needsSyncing = needsSyncing;
}
}
}
if (layer.dependencies) {
+6 -2
View File
@@ -5,10 +5,10 @@ export const devEntryPoint = join(sourceDir, "entryPoints", "dev.js");
export const deployEntryPoint = join(sourceDir, "entryPoints", "deploy.js");
export const telemetryEntryPoint = join(sourceDir, "entryPoints", "loader.js");
export const indexerEntryPoint = join(sourceDir, "entryPoints", "indexer.js");
export const deployIndexerEntryPoint = join(sourceDir, "entryPoints", "deploy-indexer.js");
export const devEntryPoints = [devEntryPoint, indexerEntryPoint, telemetryEntryPoint];
export const deployEntryPoints = [devEntryPoint, deployEntryPoint, telemetryEntryPoint];
export const deployEntryPoints = [deployIndexerEntryPoint, deployEntryPoint, telemetryEntryPoint];
export const esmShimPath = join(sourceDir, "shims", "esm.js");
@@ -22,6 +22,10 @@ export function isIndexerEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "indexer.js"));
}
export function isDeployIndexerEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy-indexer.js"));
}
export function isDeployEntryPoint(entryPoint: string) {
return entryPoint.includes(join("dist", "esm", "entryPoints", "deploy.js"));
}
+2
View File
@@ -8,6 +8,7 @@ import { COMMAND_NAME } from "../consts.js";
import { configureListProfilesCommand } from "../commands/list-profiles.js";
import { configureUpdateCommand } from "../commands/update.js";
import { VERSION } from "../version.js";
import { configureDeployCommand } from "../commands/deploy.js";
export const program = new Command();
@@ -19,6 +20,7 @@ program
configureLoginCommand(program);
configureInitCommand(program);
configureDevCommand(program);
configureDeployCommand(program);
configureWhoamiCommand(program);
configureLogoutCommand(program);
configureListProfilesCommand(program);
+593
View File
@@ -0,0 +1,593 @@
import { intro, log, outro } from "@clack/prompts";
import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build";
import {
BuildManifest,
InitializeDeploymentResponseBody,
TaskFile,
} from "@trigger.dev/core/v3/schemas";
import { CORE_VERSION } from "@trigger.dev/core/v3";
import { Command, Option as CommandOption } from "commander";
import { join, relative, resolve } from "node:path";
import { readPackageJSON, writePackageJSON } from "pkg-types";
import { z } from "zod";
import { bundleWorker } from "../build/bundle.js";
import {
createBuildContext,
notifyExtensionOnBuildComplete,
notifyExtensionOnBuildStart,
resolvePluginsForContext,
} from "../build/extensions.js";
import { createExternalsBuildExtension } from "../build/externals.js";
import {
deployEntryPoint,
deployIndexerEntryPoint,
telemetryEntryPoint,
} from "../build/packageModules.js";
import {
CommonCommandOptions,
commonOptions,
handleTelemetry,
SkipLoggingError,
wrapCommandAction,
} from "../cli/common.js";
import { loadConfig } from "../config.js";
import { createTempDir, writeJSONFile } from "../utilities/fileSystem.js";
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
import { logger } from "../utilities/logger.js";
import { getProjectClient } from "../utilities/session.js";
import { getTmpDir } from "../utilities/tempDirectories.js";
import { login } from "./login.js";
import { updateTriggerPackages } from "./update.js";
import { spinner } from "../utilities/windows.js";
import { readFile, writeFile } from "node:fs/promises";
import { VERSION } from "../version.js";
import { resolveFileSources } from "../utilities/sourceFiles.js";
import { buildImage, generateContainerfile } from "../deploy/buildImage.js";
import { buildManifestToJSON } from "../utilities/buildManifest.js";
import { CliApiClient } from "../apiClient.js";
import { chalkError, chalkWarning, cliLink } from "../utilities/cliOutput.js";
import { docs, getInTouch } from "../utilities/links.js";
import {
checkLogsForErrors,
checkLogsForWarnings,
printErrors,
printWarnings,
saveLogs,
} from "../deploy/logs.js";
const DeployCommandOptions = CommonCommandOptions.extend({
dryRun: z.boolean().default(false),
skipSyncEnvVars: z.boolean().default(false),
env: z.enum(["prod", "staging"]),
loadImage: z.boolean().default(false),
buildPlatform: z.enum(["linux/amd64", "linux/arm64"]).default("linux/amd64"),
selfHosted: z.boolean().default(false),
registry: z.string().optional(),
push: z.boolean().default(false),
config: z.string().optional(),
projectRef: z.string().optional(),
apiUrl: z.string().optional(),
saveLogs: z.boolean().default(false),
skipUpdateCheck: z.boolean().default(false),
noCache: z.boolean().default(false),
});
type DeployCommandOptions = z.infer<typeof DeployCommandOptions>;
type Deployment = InitializeDeploymentResponseBody;
export function configureDeployCommand(program: Command) {
return commonOptions(
program
.command("deploy")
.description("Deploy your Trigger.dev v3 project to the cloud.")
.argument("[path]", "The path to the project", ".")
.option(
"-e, --env <env>",
"Deploy to a specific environment (currently only prod and staging are supported)",
"prod"
)
.option("--skip-update-check", "Skip checking for @trigger.dev package updates")
.option("-c, --config <config file>", "The name of the config file, found at [path]")
.option(
"-p, --project-ref <project ref>",
"The project ref. Required if there is no config file. This will override the project specified in the config file."
)
.option(
"--dry-run",
"Do a dry run of the deployment. This will not actually deploy the project, but will show you what would be deployed."
)
.option(
"--skip-sync-env-vars",
"Skip syncing environment variables when using the syncEnvVars extension."
)
)
.addOption(
new CommandOption(
"--self-hosted",
"Build and load the image using your local Docker. Use the --registry option to specify the registry to push the image to when using --self-hosted, or just use --push to push to the default registry."
).hideHelp()
)
.addOption(
new CommandOption(
"--no-cache",
"Do not use the cache when building the image. This will slow down the build process but can be useful if you are experiencing issues with the cache."
).hideHelp()
)
.addOption(
new CommandOption(
"--push",
"When using the --self-hosted flag, push the image to the default registry. (defaults to false when not using --registry)"
).hideHelp()
)
.addOption(
new CommandOption(
"--registry <registry>",
"The registry to push the image to when using --self-hosted"
).hideHelp()
)
.addOption(
new CommandOption(
"--tag <tag>",
"(Coming soon) Specify the tag to use when pushing the image to the registry"
).hideHelp()
)
.addOption(
new CommandOption("--load-image", "Load the built image into your local docker").hideHelp()
)
.addOption(
new CommandOption(
"--build-platform <platform>",
"The platform to build the deployment image for"
)
.default("linux/amd64")
.hideHelp()
)
.addOption(
new CommandOption(
"--save-logs",
"If provided, will save logs even for successful builds"
).hideHelp()
)
.action(async (path, options) => {
await handleTelemetry(async () => {
await printStandloneInitialBanner(true);
await deployCommand(path, options);
});
});
}
export async function deployCommand(dir: string, options: unknown) {
return await wrapCommandAction("deployCommand", DeployCommandOptions, options, async (opts) => {
return await _deployCommand(dir, opts);
});
}
async function _deployCommand(dir: string, options: DeployCommandOptions) {
intro("Deploying project");
if (!options.skipUpdateCheck) {
await updateTriggerPackages(dir, { ...options }, true, true);
}
const projectPath = resolve(process.cwd(), dir);
const authorization = await login({
embedded: true,
defaultApiUrl: options.apiUrl,
profile: options.profile,
});
if (!authorization.ok) {
if (authorization.error === "fetch failed") {
throw new Error(
`Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?`
);
} else {
throw new Error(
`You must login first. Use the \`login\` CLI command.\n\n${authorization.error}`
);
}
}
const resolvedConfig = await loadConfig({
cwd: projectPath,
overrides: { project: options.projectRef },
});
logger.debug("Resolved config", resolvedConfig);
const projectClient = await getProjectClient({
accessToken: authorization.auth.accessToken,
apiUrl: authorization.auth.apiUrl,
projectRef: resolvedConfig.project,
env: options.env,
profile: options.profile,
});
if (!projectClient) {
throw new Error("Failed to get project client");
}
const serverEnvVars = await projectClient.client.getEnvironmentVariables(resolvedConfig.project);
const destination = getTmpDir(resolvedConfig.workingDir, "build", options.dryRun);
const externalsExtension = createExternalsBuildExtension("deploy", resolvedConfig);
const buildContext = createBuildContext("deploy", resolvedConfig);
buildContext.prependExtension(externalsExtension);
await notifyExtensionOnBuildStart(buildContext);
const pluginsFromExtensions = resolvePluginsForContext(buildContext);
const $buildSpinner = spinner();
$buildSpinner.start("Building project");
const bundleResult = await bundleWorker({
target: "deploy",
cwd: resolvedConfig.workingDir,
destination: destination.path,
watch: false,
resolvedConfig,
plugins: [...pluginsFromExtensions],
jsxFactory: resolvedConfig.build.jsx.factory,
jsxFragment: resolvedConfig.build.jsx.fragment,
jsxAutomatic: resolvedConfig.build.jsx.automatic,
});
$buildSpinner.stop("Successfully built project");
logger.debug("Bundle result", bundleResult);
let buildManifest: BuildManifest = {
contentHash: bundleResult.contentHash,
runtime: resolvedConfig.runtime ?? DEFAULT_RUNTIME,
environment: options.env,
packageVersion: CORE_VERSION,
cliPackageVersion: VERSION,
target: "deploy",
files: bundleResult.files,
sources: await resolveFileSources(bundleResult.files, resolvedConfig.workingDir),
config: {
project: resolvedConfig.project,
dirs: resolvedConfig.dirs,
},
outputPath: destination.path,
workerEntryPoint: bundleResult.workerEntryPoint ?? deployEntryPoint,
indexerEntryPoint: bundleResult.indexerEntryPoint ?? deployIndexerEntryPoint,
loaderEntryPoint: bundleResult.loaderEntryPoint ?? telemetryEntryPoint,
configPath: bundleResult.configPath,
deploy: {
env: serverEnvVars.success ? serverEnvVars.data.variables : {},
},
build: {},
};
buildManifest = await notifyExtensionOnBuildComplete(buildContext, buildManifest);
buildManifest = rewriteBuildManifestPaths(buildManifest, destination.path);
await writeProjectFiles(buildManifest, resolvedConfig, destination.path);
logger.debug("Successfully built project to", destination.path);
if (options.dryRun) {
logger.info(`Dry run complete. View the built project at ${destination.path}`);
return;
}
const deploymentResponse = await projectClient.client.initializeDeployment({
contentHash: buildManifest.contentHash,
userId: authorization.userId,
});
if (!deploymentResponse.success) {
throw new Error(`Failed to start deployment: ${deploymentResponse.error}`);
}
const deployment = deploymentResponse.data;
// If the deployment doesn't have any externalBuildData, then we can't use the remote image builder
// TODO: handle this and allow the user to the build and push the image themselves
if (!deployment.externalBuildData && !options.selfHosted) {
throw new Error(
`Failed to start deployment, as your instance of trigger.dev does not support hosting. To deploy this project, you must use the --self-hosted flag to build and push the image yourself.`
);
}
if (
buildManifest.deploy.env &&
Object.keys(buildManifest.deploy.env).length > 0 &&
buildManifest.deploy.needsSyncing
) {
if (!options.skipSyncEnvVars) {
const $spinner = spinner();
$spinner.start("Syncing environment variables with the server");
const success = await syncEnvVarsWithServer(
projectClient.client,
resolvedConfig.project,
options.env,
buildManifest.deploy.env
);
if (!success) {
await failDeploy(
projectClient.client,
deployment,
{
name: "SyncEnvVarsError",
message: "Failed to sync environment variables with the server",
},
"",
$spinner
);
} else {
$spinner.stop("Successfully synced environment variables with the server");
}
}
logger.log(
"Skipping syncing environment variables. The environment variables in your project have changed, but the --skip-sync-env-vars flag was provided."
);
}
const version = deployment.version;
const deploymentLink = cliLink(
"View deployment",
`${authorization.dashboardUrl}/projects/v3/${resolvedConfig.project}/deployments/${deployment.shortCode}`
);
const testLink = cliLink(
"Test tasks",
`${authorization.dashboardUrl}/projects/v3/${resolvedConfig.project}/test?environment=${
options.env === "prod" ? "prod" : "stg"
}`
);
const $spinner = spinner();
$spinner.start(`Deploying version ${version} ${deploymentLink}`);
const selfHostedRegistryHost = deployment.registryHost ?? options.registry;
const registryHost = selfHostedRegistryHost ?? "registry.trigger.dev";
const buildResult = await buildImage({
selfHosted: options.selfHosted,
buildPlatform: options.buildPlatform,
noCache: options.noCache,
push: options.push,
registryHost,
deploymentId: deployment.id,
deploymentVersion: deployment.version,
imageTag: deployment.imageTag,
contentHash: deployment.contentHash,
externalBuildId: deployment.externalBuildData?.buildId,
externalBuildToken: deployment.externalBuildData?.buildToken,
externalBuildProjectId: deployment.externalBuildData?.projectId,
projectId: projectClient.id,
projectRef: resolvedConfig.project,
apiUrl: projectClient.client.apiURL,
apiKey: projectClient.client.accessToken!,
authAccessToken: authorization.auth.accessToken,
compilationPath: destination.path,
});
logger.debug("Build result", buildResult);
const warnings = checkLogsForWarnings(buildResult.logs);
if (!warnings.ok) {
await failDeploy(
projectClient.client,
deployment,
{ name: "BuildError", message: warnings.summary },
buildResult.logs,
$spinner,
warnings.warnings,
warnings.errors
);
throw new SkipLoggingError("Failed to build image");
}
if (!buildResult.ok) {
await failDeploy(
projectClient.client,
deployment,
{ name: "BuildError", message: buildResult.error },
buildResult.logs,
$spinner,
warnings.warnings
);
throw new SkipLoggingError("Failed to build image");
}
const getDeploymentResponse = await projectClient.client.getDeployment(deployment.id);
if (!getDeploymentResponse.success) {
await failDeploy(
projectClient.client,
deployment,
{ name: "DeploymentError", message: getDeploymentResponse.error },
buildResult.logs,
$spinner
);
throw new SkipLoggingError("Failed to get deployment with worker");
}
const deploymentWithWorker = getDeploymentResponse.data;
if (!deploymentWithWorker.worker) {
await failDeploy(
projectClient.client,
deployment,
{ name: "DeploymentError", message: "Failed to get deployment with worker" },
buildResult.logs,
$spinner
);
throw new SkipLoggingError("Failed to get deployment with worker");
}
const imageReference = options.selfHosted
? `${selfHostedRegistryHost ? `${selfHostedRegistryHost}/` : ""}${buildResult.image}${
buildResult.digest ? `@${buildResult.digest}` : ""
}`
: `${registryHost}/${buildResult.image}${buildResult.digest ? `@${buildResult.digest}` : ""}`;
const finalizeResponse = await projectClient.client.finalizeDeployment(deployment.id, {
imageReference,
selfHosted: options.selfHosted,
});
if (!finalizeResponse.success) {
await failDeploy(
projectClient.client,
deployment,
{ name: "FinalizeError", message: finalizeResponse.error },
buildResult.logs,
$spinner
);
throw new SkipLoggingError("Failed to finalize deployment");
}
$spinner.stop(`Successfully deployed version ${version}`);
const taskCount = deploymentWithWorker.worker?.tasks.length ?? 0;
outro(
`Version ${version} deployed with ${taskCount} detected task${
taskCount === 1 ? "" : "s"
} | ${deploymentLink} | ${testLink}`
);
}
function rewriteBuildManifestPaths(
buildManifest: BuildManifest,
destinationDir: string
): BuildManifest {
return {
...buildManifest,
files: buildManifest.files.map((file) => ({
...file,
entry: cleanEntryPath(file.entry),
out: rewriteOutputPath(destinationDir, file.out),
})),
outputPath: rewriteOutputPath(destinationDir, buildManifest.outputPath),
configPath: rewriteOutputPath(destinationDir, buildManifest.configPath),
workerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.workerEntryPoint),
indexerEntryPoint: rewriteOutputPath(destinationDir, buildManifest.indexerEntryPoint),
loaderEntryPoint: buildManifest.loaderEntryPoint
? rewriteOutputPath(destinationDir, buildManifest.loaderEntryPoint)
: undefined,
};
}
async function writeProjectFiles(
buildManifest: BuildManifest,
resolvedConfig: ResolvedConfig,
outputPath: string
) {
// Step 1. Read the package.json file
const packageJson = await readProjectPackageJson(resolvedConfig.packageJsonPath);
if (!packageJson) {
throw new Error("Could not read the package.json file");
}
const dependencies =
buildManifest.externals?.reduce(
(acc, external) => {
acc[external.name] = external.version;
return acc;
},
{} as Record<string, string>
) ?? {};
// Step 3: Write the resolved dependencies to the package.json file
await writePackageJSON(join(outputPath, "package.json"), {
...packageJson,
name: packageJson.name ?? "trigger-project",
dependencies: {
...dependencies,
},
devDependencies: {},
peerDependencies: {},
scripts: {},
});
await writeJSONFile(join(outputPath, "build.json"), buildManifestToJSON(buildManifest));
await writeContainerfile(outputPath, buildManifest);
}
async function readProjectPackageJson(packageJsonPath: string) {
const packageJson = await readPackageJSON(packageJsonPath);
return packageJson;
}
// Remove any query parameters from the entry path
// For example, src/trigger/ai.ts?sentryProxyModule=true -> src/trigger/ai.ts
function cleanEntryPath(entry: string): string {
return entry.split("?")[0]!;
}
function rewriteOutputPath(destinationDir: string, filePath: string) {
return `/app/${relative(destinationDir, filePath)}`;
}
async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) {
const containerfile = await generateContainerfile(buildManifest);
await writeFile(join(outputPath, "Containerfile"), containerfile);
}
export async function syncEnvVarsWithServer(
apiClient: CliApiClient,
projectRef: string,
environmentSlug: string,
envVars: Record<string, string>
) {
const uploadResult = await apiClient.importEnvVars(projectRef, environmentSlug, {
variables: envVars,
override: true,
});
return uploadResult.success;
}
async function failDeploy(
client: CliApiClient,
deployment: Deployment,
error: { name: string; message: string },
logs: string,
$spinner: ReturnType<typeof spinner>,
warnings?: string[],
errors?: string[]
) {
$spinner.stop(`Failed to deploy project`);
// If there are logs, let's write it out to a temporary file and include the path in the error message
if (logs.trim() !== "") {
const logPath = await saveLogs(deployment.shortCode, logs);
printWarnings(warnings);
printErrors(errors);
checkLogsForErrors(logs);
outro(
`${chalkError("Error:")} ${error.message}. Full build logs have been saved to ${logPath}`
);
} else {
outro(`${chalkError("Error:")} ${error.message}.`);
}
await client.failDeployment(deployment.id, {
error,
});
throw new SkipLoggingError(`Failed to deploy: ${error.message}`);
}
+2 -11
View File
@@ -9,7 +9,7 @@ import Dev from "../dev/dev.js";
import { chalkError } from "../utilities/cliOutput.js";
import { printDevBanner, printStandloneInitialBanner } from "../utilities/initialBanner.js";
import { logger } from "../utilities/logger.js";
import { runtimeCheck } from "../utilities/runtimeCheck.js";
import { runtimeChecks } from "../utilities/runtimeCheck.js";
import { getProjectClient, isLoggedIn, LoginResultOk } from "../utilities/session.js";
import { updateTriggerPackages } from "./update.js";
@@ -42,17 +42,8 @@ export function configureDevCommand(program: Command) {
});
}
const MINIMUM_NODE_MAJOR = 18;
const MINIMUM_NODE_MINOR = 20;
export async function devCommand(options: DevCommandOptions) {
try {
runtimeCheck(MINIMUM_NODE_MAJOR, MINIMUM_NODE_MINOR);
} catch (e) {
logger.log(`${chalkError("X Error:")} ${e}`);
process.exitCode = 1;
return;
}
runtimeChecks();
const authorization = await isLoggedIn(options.profile);
+12 -53
View File
@@ -4,10 +4,10 @@ import { GetProjectResponseBody, flattenAttributes } from "@trigger.dev/core/v3"
import { recordSpanException } from "@trigger.dev/core/v3/workers";
import chalk from "chalk";
import { Command } from "commander";
import { ExecaError, Options as ExecaOptions, ResultPromise as ExecaResult, execa } from "execa";
import { applyEdits, modify, findNodeAtLocation, parseTree, getNodeValue } from "jsonc-parser";
import { applyEdits, findNodeAtLocation, getNodeValue, modify, parseTree } from "jsonc-parser";
import { writeFile } from "node:fs/promises";
import { join, relative, resolve } from "node:path";
import { addDependency, detectPackageManager } from "nypm";
import { z } from "zod";
import { CliApiClient } from "../apiClient.js";
import {
@@ -20,17 +20,16 @@ import {
tracer,
wrapCommandAction,
} from "../cli/common.js";
import { loadConfig } from "../config.js";
import { CLOUD_API_URL } from "../consts.js";
import { cliLink, prettyError } from "../utilities/cliOutput.js";
import { createFileFromTemplate } from "../utilities/createFileFromTemplate.js";
import { createFile, pathExists, readFile } from "../utilities/fileSystem.js";
import { PackageManager, getUserPackageManager } from "../utilities/getUserPackageManager.js";
import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
import { logger } from "../utilities/logger.js";
import { cliRootPath } from "../utilities/resolveInternalFilePath.js";
import { login } from "./login.js";
import { spinner } from "../utilities/windows.js";
import { CLOUD_API_URL } from "../consts.js";
import { cliLink, prettyError } from "../utilities/cliOutput.js";
import { loadConfig } from "../config.js";
import { login } from "./login.js";
const InitCommandOptions = CommonCommandOptions.extend({
projectRef: z.string().optional(),
@@ -431,55 +430,21 @@ async function addConfigFileToTsConfig(dir: string, options: InitCommandOptions)
async function installPackages(dir: string, options: InitCommandOptions) {
return await tracer.startActiveSpan("installPackages", async (span) => {
const installSpinner = spinner();
const projectDir = resolve(process.cwd(), dir);
let pkgManager: PackageManager | undefined;
const installSpinner = spinner();
const packageManager = await detectPackageManager(projectDir);
try {
const projectDir = resolve(process.cwd(), dir);
pkgManager = await getUserPackageManager(projectDir);
span.setAttributes({
"cli.projectDir": projectDir,
"cli.packageManager": pkgManager,
"cli.packageManager": packageManager?.name,
"cli.tag": options.tag,
});
const userArgs = options.pkgArgs?.split(",") ?? [];
const execaOptions = { cwd: projectDir } satisfies ExecaOptions;
installSpinner.start(`Adding @trigger.dev/sdk@${options.tag}`);
let installProcess: ExecaResult<typeof execaOptions>;
let args: string[];
switch (pkgManager) {
case "npm": {
// --save-exact: pin version, e.g. 3.0.0-beta.20 instead of ^3.0.0-beta.20
args = ["install", "--save-exact", ...userArgs, `@trigger.dev/sdk@${options.tag}`];
break;
}
case "pnpm":
case "yarn": {
// pins version by default
args = ["add", ...userArgs, `@trigger.dev/sdk@${options.tag}`];
break;
}
}
installSpinner.start(`Running ${pkgManager} ${args.join(" ")}`);
installProcess = execa(pkgManager, args, execaOptions);
const handleProcessOutput = (data: Buffer) => {
logger.debug(data.toString());
};
installProcess.stderr?.on("data", handleProcessOutput);
installProcess.stdout?.on("data", handleProcessOutput);
await installProcess;
await addDependency(`@trigger.dev/sdk@${options.tag}`, { cwd: projectDir });
installSpinner.stop(`@trigger.dev/sdk@${options.tag} installed`);
@@ -497,12 +462,6 @@ async function installPackages(dir: string, options: InitCommandOptions) {
recordSpanException(span, e);
}
if (e instanceof ExecaError) {
if (pkgManager) {
e.message += ` \n\nNote: You can pass additional args to ${pkgManager} by using --pkg-args. For example: trigger.dev init --pkg-args="--workspace-root"`;
}
}
span.end();
throw e;
@@ -23,7 +23,6 @@ export function configureListProfilesCommand(program: Command) {
.option("--skip-telemetry", "Opt-out of sending telemetry")
.action(async (options) => {
await handleTelemetry(async () => {
await printInitialBanner(true);
await listProfilesCommand(options);
});
});
@@ -31,6 +30,7 @@ export function configureListProfilesCommand(program: Command) {
export async function listProfilesCommand(options: unknown) {
return await wrapCommandAction("listProfiles", ListProfilesOptions, options, async (opts) => {
await printInitialBanner(false);
return await listProfiles(opts);
});
}
+13 -19
View File
@@ -1,16 +1,15 @@
import { confirm, intro, isCancel, log, outro } from "@clack/prompts";
import { z } from "zod";
import { readJSONFile, removeFile, writeJSONFile } from "../utilities/fileSystem.js";
import { spinner } from "../utilities/windows.js";
import { CommonCommandOptions, OutroCommandError, wrapCommandAction } from "../cli/common.js";
import { Command } from "commander";
import { logger } from "../utilities/logger.js";
import { PackageJson } from "type-fest";
import { printStandloneInitialBanner, updateCheck } from "../utilities/initialBanner.js";
import { join, resolve } from "path";
import { JavascriptProject } from "../utilities/javascriptProject.js";
import { PackageManager } from "../utilities/getUserPackageManager.js";
import { detectPackageManager, installDependencies } from "nypm";
import { resolve } from "path";
import { PackageJson, readPackageJSON, resolvePackageJSON } from "pkg-types";
import { z } from "zod";
import { CommonCommandOptions, OutroCommandError, wrapCommandAction } from "../cli/common.js";
import { chalkError, prettyError, prettyWarning } from "../utilities/cliOutput.js";
import { removeFile, writeJSONFile } from "../utilities/fileSystem.js";
import { printStandloneInitialBanner, updateCheck } from "../utilities/initialBanner.js";
import { logger } from "../utilities/logger.js";
import { spinner } from "../utilities/windows.js";
import { VERSION } from "../version.js";
export const UpdateCommandOptions = CommonCommandOptions.pick({
@@ -233,16 +232,12 @@ export async function updateTriggerPackages(
installSpinner.message("Installing new package versions");
const jsProject = new JavascriptProject(projectPath);
let packageManager: PackageManager | undefined;
const packageManager = await detectPackageManager(projectPath);
try {
packageManager = await jsProject.getPackageManager();
installSpinner.message(`Installing new package versions with ${packageManager}`);
await jsProject.install();
await installDependencies({ cwd: projectPath });
} catch (error) {
installSpinner.stop(
`Failed to install new package versions${packageManager ? ` with ${packageManager}` : ""}`
@@ -351,9 +346,8 @@ async function updateConfirmation(depsToUpdate: Dependency[], targetVersion: str
}
export async function getPackageJson(absoluteProjectPath: string) {
const packageJsonPath = join(absoluteProjectPath, "package.json");
const readonlyPackageJson = Object.freeze((await readJSONFile(packageJsonPath)) as PackageJson);
const packageJsonPath = await resolvePackageJSON(absoluteProjectPath);
const readonlyPackageJson = await readPackageJSON(packageJsonPath);
const packageJson = structuredClone(readonlyPackageJson);
+9
View File
@@ -241,6 +241,15 @@ function validateConfig(config: TriggerConfig, warn = true) {
config.tsconfig = config.tsconfigPath;
}
if ("resolveEnvVars" in config && typeof config.resolveEnvVars === "function") {
warn &&
logger.warn(
`The "resolveEnvVars" option is deprecated and will be removed. Use the "syncEnvVars" build extension instead. See https://trigger.dev/docs/trigger-config#syncEnvVars for more information.`
);
//
}
if (config.runtime && config.runtime === "bun") {
warn &&
logger.warn(`The "bun" runtime is currently experimental and may not work as expected.`);
+514
View File
@@ -0,0 +1,514 @@
import { join } from "node:path";
import { createTempDir, writeJSONFile } from "../utilities/fileSystem.js";
import { logger } from "../utilities/logger.js";
import { depot } from "@depot/cli";
import { x } from "tinyexec";
import { BuildManifest } from "@trigger.dev/core/v3/schemas";
export interface BuildImageOptions {
// Common options
selfHosted: boolean;
buildPlatform: string;
noCache?: boolean;
// Self-hosted specific options
push: boolean;
registry?: string;
// Non-self-hosted specific options
loadImage?: boolean;
// Flattened properties from nested structures
registryHost: string;
authAccessToken: string;
imageTag: string;
deploymentId: string;
deploymentVersion: string;
contentHash: string;
externalBuildId?: string;
externalBuildToken?: string;
externalBuildProjectId?: string;
compilationPath: string;
projectId: string;
projectRef: string;
extraCACerts?: string;
apiUrl: string;
apiKey: string;
// Optional deployment spinner
deploymentSpinner?: any; // Replace 'any' with the actual type if known
}
export async function buildImage(options: BuildImageOptions) {
const {
selfHosted,
buildPlatform,
noCache,
push,
registry,
loadImage,
registryHost,
authAccessToken,
imageTag,
deploymentId,
deploymentVersion,
contentHash,
externalBuildId,
externalBuildToken,
externalBuildProjectId,
compilationPath,
projectId,
projectRef,
extraCACerts,
apiUrl,
apiKey,
} = options;
if (selfHosted) {
return selfHostedBuildImage({
registryHost: registryHost,
imageTag: imageTag,
cwd: compilationPath,
projectId: projectId,
deploymentId: deploymentId,
deploymentVersion: deploymentVersion,
contentHash: contentHash,
projectRef: projectRef,
buildPlatform: buildPlatform,
pushImage: push,
selfHostedRegistry: !!registry,
noCache: noCache,
extraCACerts: extraCACerts,
apiUrl,
apiKey,
});
}
if (!externalBuildId || !externalBuildToken || !externalBuildProjectId) {
throw new Error(
"Failed to initialize deployment. The deployment does not have any external build data. To deploy this project, you must use the --self-hosted flag to build and push the image yourself."
);
}
return depotBuildImage({
registryHost,
auth: authAccessToken,
imageTag,
buildId: externalBuildId,
buildToken: externalBuildToken,
buildProjectId: externalBuildProjectId,
cwd: compilationPath,
projectId,
deploymentId,
deploymentVersion,
contentHash,
projectRef,
loadImage,
buildPlatform,
noCache,
extraCACerts,
apiUrl,
apiKey,
});
}
export interface DepotBuildImageOptions {
registryHost: string;
auth: string;
imageTag: string;
buildId: string;
buildToken: string;
buildProjectId: string;
cwd: string;
projectId: string;
deploymentId: string;
deploymentVersion: string;
contentHash: string;
projectRef: string;
buildPlatform: string;
apiUrl: string;
apiKey: string;
loadImage?: boolean;
noCache?: boolean;
extraCACerts?: string;
}
type BuildImageSuccess = {
ok: true;
image: string;
logs: string;
digest?: string;
};
type BuildImageFailure = {
ok: false;
error: string;
logs: string;
};
type BuildImageResults = BuildImageSuccess | BuildImageFailure;
async function depotBuildImage(options: DepotBuildImageOptions): Promise<BuildImageResults> {
// Step 3: Ensure we are "logged in" to our registry by writing to $HOME/.docker/config.json
// TODO: make sure this works on windows
const dockerConfigDir = await ensureLoggedIntoDockerRegistry(options.registryHost, {
username: "trigger",
password: options.auth,
});
const args = [
"build",
"-f",
"Containerfile",
options.noCache ? "--no-cache" : undefined,
"--platform",
options.buildPlatform,
"--provenance",
"false",
"--build-arg",
`TRIGGER_PROJECT_ID=${options.projectId}`,
"--build-arg",
`TRIGGER_DEPLOYMENT_ID=${options.deploymentId}`,
"--build-arg",
`TRIGGER_DEPLOYMENT_VERSION=${options.deploymentVersion}`,
"--build-arg",
`TRIGGER_CONTENT_HASH=${options.contentHash}`,
"--build-arg",
`TRIGGER_PROJECT_REF=${options.projectRef}`,
"--build-arg",
`TRIGGER_API_URL=${options.apiUrl}`,
"--build-arg",
`TRIGGER_SECRET_KEY=${options.apiKey}`,
...(options.extraCACerts ? ["--build-arg", `NODE_EXTRA_CA_CERTS=${options.extraCACerts}`] : []),
"--progress",
"plain",
"-t",
`${options.registryHost}/${options.imageTag}`,
".",
"--push",
options.loadImage ? "--load" : undefined,
].filter(Boolean) as string[];
logger.debug(`depot ${args.join(" ")}`);
// Step 4: Build and push the image
const childProcess = depot(args, {
cwd: options.cwd,
env: {
DEPOT_BUILD_ID: options.buildId,
DEPOT_TOKEN: options.buildToken,
DEPOT_PROJECT_ID: options.buildProjectId,
DEPOT_NO_SUMMARY_LINK: "1",
DEPOT_NO_UPDATE_NOTIFIER: "1",
DOCKER_CONFIG: dockerConfigDir,
},
});
const errors: string[] = [];
try {
const processCode = await new Promise<number | null>((res, rej) => {
// For some reason everything is output on stderr, not stdout
childProcess.stderr?.on("data", (data: Buffer) => {
const text = data.toString();
// Emitted data chunks can contain multiple lines. Remove empty lines.
const lines = text.split("\n").filter(Boolean);
errors.push(...lines);
logger.debug(text);
});
childProcess.on("error", (e) => rej(e));
childProcess.on("close", (code) => res(code));
});
const logs = extractLogs(errors);
if (processCode !== 0) {
return {
ok: false as const,
error: `Error building image`,
logs,
};
}
const digest = extractImageDigest(errors);
return {
ok: true as const,
image: options.imageTag,
logs,
digest,
};
} catch (e) {
return {
ok: false as const,
error: e instanceof Error ? e.message : JSON.stringify(e),
logs: extractLogs(errors),
};
}
}
interface SelfHostedBuildImageOptions {
registryHost: string;
imageTag: string;
cwd: string;
projectId: string;
deploymentId: string;
deploymentVersion: string;
contentHash: string;
projectRef: string;
buildPlatform: string;
pushImage: boolean;
selfHostedRegistry: boolean;
apiUrl: string;
apiKey: string;
noCache?: boolean;
extraCACerts?: string;
}
async function selfHostedBuildImage(
options: SelfHostedBuildImageOptions
): Promise<BuildImageResults> {
const imageRef = `${options.registryHost ? `${options.registryHost}/` : ""}${options.imageTag}`;
const buildArgs = [
"build",
"-f",
"Containerfile",
options.noCache ? "--no-cache" : undefined,
"--platform",
options.buildPlatform,
"--build-arg",
`TRIGGER_PROJECT_ID=${options.projectId}`,
"--build-arg",
`TRIGGER_DEPLOYMENT_ID=${options.deploymentId}`,
"--build-arg",
`TRIGGER_DEPLOYMENT_VERSION=${options.deploymentVersion}`,
"--build-arg",
`TRIGGER_CONTENT_HASH=${options.contentHash}`,
"--build-arg",
`TRIGGER_PROJECT_REF=${options.projectRef}`,
"--build-arg",
`TRIGGER_API_URL=${options.apiUrl}`,
"--build-arg",
`TRIGGER_SECRET_KEY=${options.apiKey}`,
...(options.extraCACerts ? ["--build-arg", `NODE_EXTRA_CA_CERTS=${options.extraCACerts}`] : []),
"--progress",
"plain",
"-t",
imageRef,
".", // The build context
].filter(Boolean) as string[];
logger.debug(`docker ${buildArgs.join(" ")}`, {
cwd: options.cwd,
});
const errors: string[] = [];
let digest: string | undefined;
// Build the image
const buildProcess = x("docker", buildArgs, {
nodeOptions: { cwd: options.cwd },
});
for await (const line of buildProcess) {
// line will be from stderr/stdout in the order you'd see it in a term
errors.push(line);
logger.debug(line);
}
if (buildProcess.exitCode !== 0) {
return {
ok: false as const,
error: "Error building image",
logs: extractLogs(errors),
};
}
digest = extractImageDigest(errors);
if (options.selfHostedRegistry || options.pushImage) {
const pushArgs = ["push", imageRef].filter(Boolean) as string[];
logger.debug(`docker ${pushArgs.join(" ")}`);
// Push the image
const pushProcess = x("docker", pushArgs, {
nodeOptions: { cwd: options.cwd },
});
for await (const line of pushProcess) {
logger.debug(line);
}
if (pushProcess.exitCode !== 0) {
return {
ok: false as const,
error: "Error pushing image",
logs: extractLogs(errors),
};
}
}
return {
ok: true as const,
image: options.imageTag,
digest,
logs: extractLogs(errors),
};
}
async function ensureLoggedIntoDockerRegistry(
registryHost: string,
auth: { username: string; password: string }
) {
const tmpDir = await createTempDir();
// Read the current docker config
const dockerConfigPath = join(tmpDir, "config.json");
await writeJSONFile(dockerConfigPath, {
auths: {
[registryHost]: {
auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"),
},
},
});
logger.debug(`Writing docker config to ${dockerConfigPath}`);
return tmpDir;
}
function extractLogs(outputs: string[]) {
// Remove empty lines
const cleanedOutputs = outputs.map((line) => line.trim()).filter((line) => line !== "");
return cleanedOutputs.map((line) => line.trim()).join("\n");
}
function extractImageDigest(outputs: string[]) {
const imageDigestRegex = /pushing manifest for .+(?<digest>sha256:[a-f0-9]{64})/;
for (const line of outputs) {
const imageDigestMatch = line.match(imageDigestRegex);
const digest = imageDigestMatch?.groups?.digest;
if (digest) {
return digest;
}
}
return;
}
export async function generateContainerfile(buildManifest: BuildManifest) {
switch (buildManifest.runtime) {
case "node20": {
return await generateNodeContainerfile(buildManifest);
}
case "bun": {
return await generateBunContainerfile(buildManifest);
}
}
}
async function generateBunContainerfile(buildManifest: BuildManifest) {
return "";
}
async function generateNodeContainerfile(buildManifest: BuildManifest) {
const buildArgs = Object.entries(buildManifest.build.env || {})
.flatMap(([key]) => `ARG ${key}`)
.join("\n");
const buildEnvVars = Object.entries(buildManifest.build.env || {})
.flatMap(([key]) => `ENV ${key}=$${key}`)
.join("\n");
const postInstallCommands = (buildManifest.build.commands || [])
.map((cmd) => `RUN ${cmd}`)
.join("\n");
return `
FROM node:21-bookworm-slim@sha256:99afef5df7400a8d118e0504576d32ca700de5034c4f9271d2ff7c91cc12d170 AS base
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get --fix-broken install -y && apt-get install -y --no-install-recommends busybox ca-certificates dumb-init git openssl && apt-get clean && rm -rf /var/lib/apt/lists/*
FROM base AS install
USER node
WORKDIR /app
${buildArgs}
${buildEnvVars}
ENV NODE_ENV=production
ENV NPM_CONFIG_UPDATE_NOTIFIER=false
COPY --chown=node:node package.json ./
RUN npm i --no-audit --no-fund --no-save --no-package-lock
# Now copy all the files
# IMPORTANT: Do this after running npm install because npm i will wipe out the node_modules directory
COPY --chown=node:node . .
${postInstallCommands}
from install as indexer
USER node
WORKDIR /app
ARG TRIGGER_PROJECT_ID
ARG TRIGGER_DEPLOYMENT_ID
ARG TRIGGER_DEPLOYMENT_VERSION
ARG TRIGGER_CONTENT_HASH
ARG TRIGGER_PROJECT_REF
ARG NODE_EXTRA_CA_CERTS
ARG TRIGGER_SECRET_KEY
ARG TRIGGER_API_URL
ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
TRIGGER_DEPLOYMENT_ID=\${TRIGGER_DEPLOYMENT_ID} \
TRIGGER_DEPLOYMENT_VERSION=\${TRIGGER_DEPLOYMENT_VERSION} \
TRIGGER_PROJECT_REF=\${TRIGGER_PROJECT_REF} \
TRIGGER_CONTENT_HASH=\${TRIGGER_CONTENT_HASH} \
TRIGGER_SECRET_KEY=\${TRIGGER_SECRET_KEY} \
TRIGGER_API_URL=\${TRIGGER_API_URL} \
NODE_EXTRA_CA_CERTS=\${NODE_EXTRA_CA_CERTS} \
NODE_ENV=production \
NODE_OPTIONS="--max_old_space_size=8192"
# Run the indexer
RUN node ${buildManifest.indexerEntryPoint}
# Development or production stage builds upon the base stage
FROM base AS final
USER node
WORKDIR /app
ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
TRIGGER_DEPLOYMENT_ID=\${TRIGGER_DEPLOYMENT_ID} \
TRIGGER_DEPLOYMENT_VERSION=\${TRIGGER_DEPLOYMENT_VERSION} \
TRIGGER_CONTENT_HASH=\${TRIGGER_CONTENT_HASH} \
TRIGGER_PROJECT_REF=\${TRIGGER_PROJECT_REF} \
NODE_EXTRA_CA_CERTS=\${NODE_EXTRA_CA_CERTS} \
NODE_ENV=production \
NODE_OPTIONS="--max_old_space_size=8192"
# Copy the files from the install stage
COPY --from=install --chown=node:node /app ./
# Copy the index.json file from the indexer stage
COPY --from=indexer --chown=node:node /app/index.json ./
ENTRYPOINT [ "dumb-init", "node", "${buildManifest.workerEntryPoint}" ]
CMD []
`;
}
+147
View File
@@ -0,0 +1,147 @@
import { log } from "@clack/prompts";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { chalkError, chalkWarning, cliLink } from "../utilities/cliOutput.js";
import { createTempDir } from "../utilities/fileSystem.js";
import { docs, getInTouch } from "../utilities/links.js";
export type WarningsCheckReturn =
| {
ok: true;
warnings: string[];
}
| {
ok: false;
summary: string;
errors: string[];
warnings: string[];
};
export type LogParserOptions = Array<{
regex: RegExp;
message: string;
shouldFail?: boolean;
}>;
export async function saveLogs(shortCode: string, logs: string) {
const logPath = join(await createTempDir(), `build-${shortCode}.log`);
await writeFile(logPath, logs);
return logPath;
}
export function printErrors(errors?: string[]) {
for (const error of errors ?? []) {
log.error(`${chalkError("Error:")} ${error}`);
}
}
export function printWarnings(warnings?: string[]) {
for (const warning of warnings ?? []) {
log.warn(`${chalkWarning("Warning:")} ${warning}`);
}
}
// Try to extract useful error messages from the logs
export function checkLogsForErrors(logs: string) {
const errors: LogParserOptions = [
{
regex: /Error: Provided --schema at (?<schema>.*) doesn't exist/,
message: `Prisma generate failed to find the specified schema at "$schema".\nDid you include it in config.additionalFiles? ${cliLink(
"Config docs",
docs.config.prisma
)}`,
},
{
regex: /@prisma\/client did not initialize yet/,
message: `Prisma client not initialized yet.\nDid you forget to add the postinstall script? ${cliLink(
"Config docs",
docs.config.prisma
)}`,
},
{
regex: /sh: 1: (?<packageOrBinary>.*): not found/,
message: `$packageOrBinary not found\n\nIf it's a package: Include it in ${cliLink(
"config.additionalPackages",
docs.config.prisma
)}\nIf it's a binary: Please ${cliLink(
"get in touch",
getInTouch
)} and we'll see what we can do!`,
},
];
for (const error of errors) {
const matches = logs.match(error.regex);
if (!matches) {
continue;
}
const message = getMessageFromTemplate(error.message, matches.groups);
log.error(`${chalkError("Error:")} ${message}`);
break;
}
}
function getMessageFromTemplate(template: string, replacer: RegExpMatchArray["groups"]) {
let message = template;
if (replacer) {
for (const [key, value] of Object.entries(replacer)) {
message = message.replaceAll(`$${key}`, value);
}
}
return message;
}
// Try to extract useful warnings from logs. Sometimes we may even want to fail the build. This won't work if the step is cached.
export function checkLogsForWarnings(logs: string): WarningsCheckReturn {
const warnings: LogParserOptions = [
{
regex: /prisma:warn We could not find your Prisma schema/,
message: `Prisma generate failed to find the default schema. Did you include it in config.additionalFiles? ${cliLink(
"Config docs",
docs.config.prisma
)}\nCustom schema paths require a postinstall script like this: \`prisma generate --schema=./custom/path/to/schema.prisma\``,
shouldFail: true,
},
];
const errorMessages: string[] = [];
const warningMessages: string[] = [];
let shouldFail = false;
for (const warning of warnings) {
const matches = logs.match(warning.regex);
if (!matches) {
continue;
}
const message = getMessageFromTemplate(warning.message, matches.groups);
if (warning.shouldFail) {
shouldFail = true;
errorMessages.push(message);
} else {
warningMessages.push(message);
}
}
if (shouldFail) {
return {
ok: false,
summary: "Build succeeded with critical warnings. Will not proceed",
warnings: warningMessages,
errors: errorMessages,
};
}
return {
ok: true,
warnings: warningMessages,
};
}
+10 -1
View File
@@ -1,5 +1,5 @@
import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build";
import { BuildManifest } from "@trigger.dev/core/v3/schemas";
import { BuildManifest, TaskFile } from "@trigger.dev/core/v3/schemas";
import * as esbuild from "esbuild";
import { CliApiClient } from "../apiClient.js";
import {
@@ -24,6 +24,11 @@ import { logger } from "../utilities/logger.js";
import { EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js";
import { startDevOutput } from "./devOutput.js";
import { startWorkerRuntime } from "./workerRuntime.js";
import { VERSION } from "../version.js";
import { CORE_VERSION } from "@trigger.dev/core/v3";
import { join } from "node:path";
import { readFile } from "node:fs/promises";
import { resolveFileSources } from "../utilities/sourceFiles.js";
export type DevSessionOptions = {
name: string | undefined;
@@ -172,8 +177,12 @@ async function createBuildManifestFromBundle(
const buildManifest: BuildManifest = {
contentHash: bundle.contentHash,
runtime: resolvedConfig.runtime ?? DEFAULT_RUNTIME,
cliPackageVersion: VERSION,
packageVersion: CORE_VERSION,
environment: "dev",
target: "dev",
files: bundle.files,
sources: await resolveFileSources(bundle.files, resolvedConfig.workingDir),
externals: [],
config: {
project: resolvedConfig.project,
+18 -19
View File
@@ -18,17 +18,14 @@ import { ClientRequestArgs } from "node:http";
import { WebSocket } from "partysocket";
import { ClientOptions, WebSocket as wsWebSocket } from "ws";
import { CliApiClient } from "../apiClient.js";
import { getInstrumentedPackageNames } from "../build/instrumentation.js";
import { DevCommandOptions } from "../commands/dev.js";
import { chalkError, chalkTask } from "../utilities/cliOutput.js";
import { resolveDotEnvVars } from "../utilities/dotEnv.js";
import { eventBus } from "../utilities/eventBus.js";
import { logger } from "../utilities/logger.js";
import { VERSION } from "../version.js";
import { BackgroundWorker, BackgroundWorkerCoordinator } from "./backgroundWorker.js";
import { getInstrumentedPackageNames } from "../build/instrumentation.js";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { createHash } from "node:crypto";
import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js";
export interface WorkerRuntime {
shutdown(): Promise<void>;
@@ -186,19 +183,16 @@ class DevWorkerRuntime implements WorkerRuntime {
return;
}
const fileContents = await this.#fetchTaskFiles(
backgroundWorker.manifest.tasks,
this.options.config.workingDir
);
const sourceFiles = resolveTaskSourceFiles(manifest.sources, backgroundWorker.manifest.tasks);
const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = {
localOnly: true,
metadata: {
packageVersion: VERSION,
cliPackageVersion: VERSION,
packageVersion: manifest.packageVersion,
cliPackageVersion: manifest.cliPackageVersion,
tasks: backgroundWorker.manifest.tasks,
contentHash: manifest.contentHash,
fileContents,
sourceFiles,
},
supportsLazyAttempts: true,
};
@@ -219,7 +213,10 @@ class DevWorkerRuntime implements WorkerRuntime {
eventBus.emit("backgroundWorkerInitialized", backgroundWorker);
}
async #fetchTaskFiles(tasks: TaskManifest[], workingDir: string) {
async #fetchTaskFiles(
sources: Record<string, { contents: string; contentHash: string }>,
tasks: TaskManifest[]
) {
const tasksGroupedByFile: Record<string, TaskManifest[]> = {};
for (const task of tasks) {
@@ -238,16 +235,18 @@ class DevWorkerRuntime implements WorkerRuntime {
}> = [];
for (const [filePath, tasks] of Object.entries(tasksGroupedByFile)) {
const contents = await readFile(join(workingDir, filePath), "utf-8");
const source = sources[filePath];
if (!source) {
continue;
}
const taskIds = tasks.map((task) => task.id);
const hasher = createHash("md5");
hasher.update(contents);
taskFiles.push({
filePath,
...source,
taskIds,
contents,
contentHash: hasher.digest("hex"),
filePath,
});
}
@@ -0,0 +1,237 @@
import {
BuildManifest,
CreateBackgroundWorkerRequestBody,
type HandleErrorFunction,
taskCatalog,
TriggerConfig,
WorkerManifest,
} from "@trigger.dev/core/v3";
import {
StandardTaskCatalog,
TracingDiagnosticLogLevel,
TracingSDK,
} from "@trigger.dev/core/v3/workers";
import { readFile, writeFile } from "node:fs/promises";
import sourceMapSupport from "source-map-support";
import { CliApiClient } from "../apiClient.js";
import { resolveTaskSourceFiles } from "../utilities/sourceFiles.js";
import { join } from "node:path";
sourceMapSupport.install({
handleUncaughtExceptions: false,
environment: "node",
hookRequire: false,
});
taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog());
async function importConfig(configPath: string): Promise<{
config: TriggerConfig;
handleError?: HandleErrorFunction;
}> {
const configModule = await import(configPath);
const config = configModule?.default ?? configModule?.config;
return {
config,
handleError: configModule?.handleError,
};
}
async function loadBuildManifest() {
const manifestContents = await readFile("./build.json", "utf-8");
const raw = JSON.parse(manifestContents);
return BuildManifest.parse(raw);
}
// We need to make sure, that if any errors are thrown, that we fail the deployment
// 1. Fetch the build manifest
// 2. Fetch the environment variables from the server
// 3. Import the config
// 5. Inject the env vars into process.env
// 6. Configure the tracing SDK
// 7. Load all the tasks from the build manifest and create the index.json
// 8. Write the index.json to the file system
// 9. Update the deployment with the worker index.json
// 10. Exit the process
async function bootstrap() {
const buildManifest = await loadBuildManifest();
if (typeof process.env.TRIGGER_API_URL !== "string") {
console.error("TRIGGER_API_URL is not set");
process.exit(1);
}
const cliApiClient = new CliApiClient(
process.env.TRIGGER_API_URL,
process.env.TRIGGER_SECRET_KEY
);
if (!process.env.TRIGGER_PROJECT_REF) {
console.error("TRIGGER_PROJECT_REF is not set");
process.exit(1);
}
if (!process.env.TRIGGER_DEPLOYMENT_ID) {
console.error("TRIGGER_DEPLOYMENT_ID is not set");
process.exit(1);
}
return {
buildManifest,
cliApiClient,
projectRef: process.env.TRIGGER_PROJECT_REF,
deploymentId: process.env.TRIGGER_DEPLOYMENT_ID,
};
}
type BootstrapResult = Awaited<ReturnType<typeof bootstrap>>;
async function indexDeployment({
cliApiClient,
projectRef,
deploymentId,
buildManifest,
}: BootstrapResult) {
try {
const env = await cliApiClient.getEnvironmentVariables(projectRef);
if (!env.success) {
throw new Error(`Failed to fetch environment variables: ${env.error}`);
}
injectEnvVars(env.data.variables);
const { config } = await importConfig(buildManifest.configPath);
// This needs to run or the PrismaInstrumentation will throw an error
new TracingSDK({
url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
instrumentations: config.instrumentations ?? [],
diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
});
const importErrors: Array<{ error: Error; file: string }> = [];
for (const file of buildManifest.files) {
const [error, module] = await $import(file.out);
if (error) {
importErrors.push({ error, file: file.entry });
continue;
}
for (const exportName of getExportNames(module)) {
const task = module[exportName] ?? module.default?.[exportName];
if (!task) {
continue;
}
if (task[Symbol.for("trigger.dev/task")]) {
if (taskCatalog.taskExists(task.id)) {
taskCatalog.registerTaskFileMetadata(task.id, {
exportName,
filePath: file.entry,
entryPoint: file.out,
});
}
}
}
}
console.log("Import errors", importErrors);
if (importErrors.length > 0) {
const errorMessages = importErrors.map((error) => {
return `${error.file}: ${error.error.message}`;
});
throw new Error(`Failed to index task files:\n${errorMessages.join("\n")}`);
}
const tasks = taskCatalog.listTaskManifests();
const workerManifest: WorkerManifest = { tasks, configPath: buildManifest.configPath };
console.log("Writing index.json", process.cwd());
await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2));
const sourceFiles = resolveTaskSourceFiles(buildManifest.sources, workerManifest.tasks);
const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = {
localOnly: true,
metadata: {
contentHash: buildManifest.contentHash,
packageVersion: buildManifest.packageVersion,
cliPackageVersion: buildManifest.cliPackageVersion,
tasks: workerManifest.tasks,
sourceFiles,
},
supportsLazyAttempts: true,
};
await cliApiClient.createDeploymentBackgroundWorker(deploymentId, backgroundWorkerBody);
} catch (error) {
// If we have an error, we need to fail the deployment
await cliApiClient.failDeployment(deploymentId, {
error:
error instanceof Error
? {
name: error.name,
message: error.message,
stack: error.stack,
}
: {
name: "Error",
message: String(error),
},
});
throw error;
}
}
const results = await bootstrap();
await indexDeployment(results);
function getExportNames(module: any) {
const exports: string[] = [];
const exportKeys = Object.keys(module);
if (exportKeys.length === 0) {
return exports;
}
if (exportKeys.length === 1 && exportKeys[0] === "default") {
return Object.keys(module.default);
}
return exportKeys;
}
type Result<T> = [Error | null, T | null];
async function $import(path: string): Promise<Result<any>> {
try {
const module = await import(path);
return [null, module];
} catch (error) {
return [error as Error, null];
}
}
function injectEnvVars(env: Record<string, string>) {
for (const [key, value] of Object.entries(env)) {
process.env[key] = value;
}
}
@@ -0,0 +1,11 @@
import { BuildManifest } from "@trigger.dev/core/v3/schemas";
export function buildManifestToJSON(manifest: BuildManifest): BuildManifest {
const { deploy, build, ...rest } = manifest;
return {
...rest,
deploy: {},
build: {},
};
}
@@ -2,6 +2,8 @@ import { log } from "@clack/prompts";
import chalk from "chalk";
import terminalLink, { Options as TerminalLinkOptions } from "terminal-link";
export const isInteractive = process.stdin.isTTY;
export const green = "#4FFF54";
export const purple = "#735BF3";
@@ -1,72 +0,0 @@
import { findUp } from "find-up";
import { basename } from "path";
import { logger } from "./logger.js";
export type PackageManager = "npm" | "pnpm" | "yarn";
export const LOCKFILES = {
npm: "package-lock.json",
npmShrinkwrap: "npm-shrinkwrap.json",
pnpm: "pnpm-lock.yaml",
yarn: "yarn.lock",
bun: "bun.lockb",
};
export async function getUserPackageManager(path: string): Promise<PackageManager> {
const packageManager = await detectPackageManager(path);
logger.debug("Detected package manager", { packageManager });
return packageManager;
}
async function detectPackageManager(path: string): Promise<PackageManager> {
try {
return await detectPackageManagerFromArtifacts(path);
} catch (error) {
return detectPackageManagerFromCurrentCommand();
}
}
function detectPackageManagerFromCurrentCommand(): PackageManager {
// This environment variable is set by npm and yarn but pnpm seems less consistent
const userAgent = process.env.npm_config_user_agent;
if (userAgent) {
if (userAgent.startsWith("yarn")) {
return "yarn";
} else if (userAgent.startsWith("pnpm")) {
return "pnpm";
} else {
return "npm";
}
} else {
// If no user agent is set, assume npm
return "npm";
}
}
export async function detectPackageManagerFromArtifacts(path: string): Promise<PackageManager> {
const foundPath = await findUp(Object.values(LOCKFILES), { cwd: path });
if (!foundPath) {
throw new Error("Could not detect package manager from artifacts");
}
logger.debug("Found path from package manager artifacts", { foundPath });
switch (basename(foundPath)) {
case LOCKFILES.yarn:
logger.debug("Found yarn artifact", { foundPath });
return "yarn";
case LOCKFILES.pnpm:
logger.debug("Found pnpm artifact", { foundPath });
return "pnpm";
case LOCKFILES.npm:
case LOCKFILES.npmShrinkwrap:
logger.debug("Found npm artifact", { foundPath });
return "npm";
case LOCKFILES.bun:
logger.debug("Found bun artifact", { foundPath });
return "npm";
default:
throw new Error(`Unhandled package manager detection path: ${foundPath}`);
}
}
+34 -15
View File
@@ -1,11 +1,9 @@
import chalk from "chalk";
import type { Result } from "update-check";
import checkForUpdate from "update-check";
import { chalkGrey, chalkRun, chalkTask, chalkWorker, green, logo } from "./cliOutput.js";
import { getLatestVersion } from "fast-npm-meta";
import { VERSION } from "../version.js";
import { chalkGrey, chalkRun, chalkTask, chalkWorker, logo } from "./cliOutput.js";
import { logger } from "./logger.js";
import { spinner } from "./windows.js";
import { readPackageJSON } from "pkg-types";
import { VERSION } from "../version.js";
export async function printInitialBanner(performUpdateCheck = true) {
const text = `\n${logo()} ${chalkGrey(`(${VERSION})`)}\n`;
@@ -14,24 +12,29 @@ export async function printInitialBanner(performUpdateCheck = true) {
let maybeNewVersion: string | undefined;
if (performUpdateCheck) {
const loadingSpinner = spinner();
loadingSpinner.start("Checking for updates");
const $spinner = spinner();
$spinner.start("Checking for updates");
maybeNewVersion = await updateCheck();
// Log a slightly more noticeable message if this is a major bump
if (maybeNewVersion !== undefined) {
loadingSpinner.stop(`Update available ${chalk.green(maybeNewVersion)}`);
$spinner.stop(`Update available ${chalk.green(maybeNewVersion)}`);
const currentMajor = parseInt(VERSION.split(".")[0]!);
const newMajor = parseInt(maybeNewVersion.split(".")[0]!);
logger.debug(`updateCheck: ${VERSION} -> ${maybeNewVersion}`);
if (newMajor > currentMajor) {
logger.warn(
`Please update to the latest version of \`trigger.dev\` to prevent critical errors.
Run \`npm install --save-dev trigger.dev@${newMajor}\` to update to the latest version.
After installation, run Trigger.dev with \`npx trigger.dev\`.`
);
} else {
}
} else {
loadingSpinner.stop("On latest version");
$spinner.stop("On latest version");
}
}
}
@@ -67,17 +70,33 @@ export function printDevBanner(printTopBorder = true) {
}
async function doUpdateCheck(): Promise<string | undefined> {
let update: Result | null = null;
try {
const pkg = await readPackageJSON();
// default cache for update check is 1 day
update = await checkForUpdate.default(pkg, {
distTag: VERSION.startsWith("3.0.0-beta") ? "beta" : "latest",
});
const meta = await getLatestVersion(
`trigger.dev@${VERSION.startsWith("3.0.0-beta") ? "beta" : "latest"}`,
{ force: true }
);
if (!meta.version) {
return;
}
const compareVersions = (a: string, b: string) =>
a.localeCompare(b, "en-US", { numeric: true });
const comparison = compareVersions(VERSION, meta.version);
if (comparison === -1) {
return meta.version;
}
return;
} catch (err) {
// ignore error
logger.debug(err);
return;
}
return update?.latest;
}
//only do this once while the cli is running
@@ -1,706 +0,0 @@
import { $, ExecaError } from "execa";
import { join } from "node:path";
import { readJSONFileSync } from "./fileSystem.js";
import { logger } from "./logger.js";
import { PackageManager, getUserPackageManager } from "./getUserPackageManager.js";
import { PackageJson } from "type-fest";
import { assertExhaustive } from "./assertExhaustive.js";
import { builtinModules } from "node:module";
import { tracer } from "../cli/common.js";
import { recordSpanException } from "@trigger.dev/core/v3/otel";
import { flattenAttributes } from "@trigger.dev/core/v3";
export type ResolveOptions = { allowDev: boolean };
export type DependencyMeta = { version: string; external: boolean };
export class JavascriptProject {
private _packageJson?: PackageJson;
private _packageManager?: PackageManager;
constructor(private projectPath: string) {}
private get packageJson() {
if (!this._packageJson) {
this._packageJson = readJSONFileSync(join(this.projectPath, "package.json")) as PackageJson;
}
return this._packageJson;
}
public get allowedPackageJson(): Record<string, unknown> {
const disallowedKeys = [
"scripts",
"devDependencies",
"dependencies",
"peerDependencies",
"author",
"contributors",
"funding",
"bugs",
"files",
"keywords",
"main",
"module",
"type",
"bin",
"browser",
"man",
"directories",
"repository",
"peerDependenciesMeta",
"optionalDependencies",
"engines",
"os",
"cpu",
"private",
"publishConfig",
"workspaces",
];
return Object.keys(this.packageJson).reduce(
(acc, key) => {
if (!disallowedKeys.includes(key)) {
acc[key] = this.packageJson[key];
}
return acc;
},
{} as Record<string, unknown>
);
}
public get scripts(): Record<string, string> {
return this.#filterScripts();
}
#filterScripts(): Record<string, string> {
if (!this.packageJson.scripts || typeof this.packageJson.scripts !== "object") {
return {};
}
return this.packageJson.scripts as Record<string, string>;
}
async install(): Promise<void> {
const command = await this.#getCommand();
try {
await command.installDependencies({
cwd: this.projectPath,
});
} catch (error) {
logger.debug(`Failed to install dependencies using ${command.name}`, {
error,
});
}
}
async extractDirectDependenciesMeta(): Promise<Record<string, DependencyMeta>> {
return tracer.startActiveSpan(
"JavascriptProject.extractDirectDependenciesMeta",
async (span) => {
const command = await this.#getCommand();
span.setAttributes({
packageManager: command.name,
});
try {
span.end();
return await command.extractDirectDependenciesMeta({
cwd: this.projectPath,
});
} catch (error) {
recordSpanException(span, error);
span.end();
logger.debug(`Failed to resolve internal dependencies using ${command.name}`, {
error,
});
throw error;
}
}
);
}
async resolveAll(packageNames: string[]): Promise<Record<string, string>> {
return tracer.startActiveSpan("JavascriptProject.resolveAll", async (span) => {
const externalPackages = packageNames.filter((packageName) => !isBuiltInModule(packageName));
const command = await this.#getCommand();
span.setAttributes({
externalPackages,
packageManager: command.name,
});
try {
const versions = await command.resolveDependencyVersions(externalPackages, {
cwd: this.projectPath,
});
if (versions) {
logger.debug(`Resolved [${externalPackages.join(", ")}] version using ${command.name}`, {
versions,
});
span.setAttributes({
...flattenAttributes(versions, "versions"),
});
}
// Merge the resolved versions with the package.json dependencies
const missingPackages = externalPackages.filter((packageName) => !versions[packageName]);
const missingPackageVersions: Record<string, string> = {};
for (const packageName of missingPackages) {
const packageJsonVersion = this.packageJson.dependencies?.[packageName];
if (typeof packageJsonVersion === "string") {
logger.debug(`Resolved ${packageName} version using package.json`, {
packageJsonVersion,
});
missingPackageVersions[packageName] = packageJsonVersion;
}
}
span.setAttributes({
...flattenAttributes(missingPackageVersions, "missingPackageVersions"),
missingPackages,
});
span.end();
return { ...versions, ...missingPackageVersions };
} catch (error) {
recordSpanException(span, error);
span.end();
logger.debug(`Failed to resolve dependency versions using ${command.name}`, {
packageNames,
error,
});
return {};
}
});
}
async resolve(packageName: string, options?: ResolveOptions): Promise<string | undefined> {
if (isBuiltInModule(packageName)) {
return undefined;
}
const opts = { allowDev: false, ...options };
const command = await this.#getCommand();
try {
const version = await command.resolveDependencyVersion(packageName, {
cwd: this.projectPath,
});
if (version) {
logger.debug(`Resolved ${packageName} version using ${command.name}`, { version });
return version;
}
const packageJsonVersion = this.packageJson.dependencies?.[packageName];
if (typeof packageJsonVersion === "string") {
logger.debug(`Resolved ${packageName} version using package.json`, { packageJsonVersion });
return packageJsonVersion;
}
if (opts.allowDev) {
const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
if (typeof devPackageJsonVersion === "string") {
logger.debug(`Resolved ${packageName} version using devDependencies`, {
devPackageJsonVersion,
});
return devPackageJsonVersion;
}
}
} catch (error) {
logger.debug(`Failed to resolve dependency version using ${command.name}`, {
packageName,
error,
});
}
return;
}
async #getCommand(): Promise<PackageManagerCommands> {
const packageManager = await this.getPackageManager();
switch (packageManager) {
case "npm":
return new NPMCommands();
case "pnpm":
return new PNPMCommands();
case "yarn":
return new YarnCommands();
default:
assertExhaustive(packageManager);
}
}
async getPackageManager(): Promise<PackageManager> {
if (!this._packageManager) {
this._packageManager = await getUserPackageManager(this.projectPath);
}
return this._packageManager;
}
}
type PnpmList = {
name: string;
path: string;
version: string;
private: boolean;
dependencies?: Record<
string,
{
from: string;
version: string;
resolved: string;
path: string;
}
>;
}[];
type PackageManagerOptions = {
cwd?: string;
};
interface PackageManagerCommands {
name: string;
installDependencies(options: PackageManagerOptions): Promise<void>;
extractDirectDependenciesMeta(
options: PackageManagerOptions
): Promise<Record<string, DependencyMeta>>;
resolveDependencyVersion(
packageName: string,
options: PackageManagerOptions
): Promise<string | undefined>;
resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>>;
}
class PNPMCommands implements PackageManagerCommands {
get name() {
return "pnpm";
}
private get cmd() {
return process.platform === "win32" ? "pnpm.cmd" : "pnpm";
}
async installDependencies(options: PackageManagerOptions) {
const { stdout, stderr } = await $({ cwd: options.cwd })`${this.cmd} install`;
logger.debug(`Installing dependencies using ${this.name}`, { stdout, stderr });
}
async resolveDependencyVersion(packageName: string, options: PackageManagerOptions) {
const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageName} -r --json`;
const result = JSON.parse(stdout) as PnpmList;
logger.debug(`Resolving ${packageName} version using ${this.name}`);
// Return the first dependency version that matches the package name
for (const dep of result) {
const dependency = dep.dependencies?.[packageName];
if (dependency) {
return dependency.version;
}
}
return;
}
async resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>> {
const result = await this.#listDependencies(packageNames, options);
logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
const results: Record<string, string> = {};
// Return the first dependency version that matches the package name
for (const dep of result) {
for (const packageName of packageNames) {
const dependency = dep.dependencies?.[packageName];
if (dependency) {
results[packageName] = dependency.version;
}
}
}
return results;
}
async extractDirectDependenciesMeta(options: PackageManagerOptions) {
const result = await this.#listDirectDependencies(options);
logger.debug(`Extracting direct dependencies metadata using ${this.name}`);
const results: Record<string, DependencyMeta> = {};
for (const projectPkg of result) {
results[projectPkg.name] = { version: projectPkg.version, external: false };
if (projectPkg.dependencies) {
for (const [name, dep] of Object.entries(projectPkg.dependencies)) {
const { version } = dep;
results[name] = {
version,
external: !version.startsWith("link:"),
};
}
}
}
return results;
}
async #listDirectDependencies(options: PackageManagerOptions) {
const childProcess = await $({
cwd: options.cwd,
reject: false,
})`${this.cmd} list --recursive --json`;
if (childProcess.failed) {
logger.debug("Failed to list dependencies, using stdout anyway...", {
error: childProcess,
});
}
return JSON.parse(childProcess.stdout) as PnpmList;
}
async #listDependencies(packageNames: string[], options: PackageManagerOptions) {
const childProcess = await $({
cwd: options.cwd,
reject: false,
})`${this.cmd} list ${packageNames} -r --json`;
if (childProcess.failed) {
logger.debug("Failed to list dependencies, using stdout anyway...", {
error: childProcess,
});
}
return JSON.parse(childProcess.stdout) as PnpmList;
}
}
type NpmDependency = {
version: string;
resolved: string;
overridden: boolean;
required?: { version: string };
dependencies?: Record<string, NpmDependency>;
};
type NpmListOutput = {
dependencies: Record<string, NpmDependency>;
};
class NPMCommands implements PackageManagerCommands {
get name() {
return "npm";
}
private get cmd() {
return process.platform === "win32" ? "npm.cmd" : "npm";
}
async installDependencies(options: PackageManagerOptions) {
const { stdout, stderr } = await $({ cwd: options.cwd })`${this.cmd} install`;
logger.debug(`Installing dependencies using ${this.name}`, { stdout, stderr });
}
async resolveDependencyVersion(packageName: string, options: PackageManagerOptions) {
const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageName} --json`;
const output = JSON.parse(stdout) as NpmListOutput;
logger.debug(`Resolving ${packageName} version using ${this.name}`, { output });
return this.#recursivelySearchDependencies(output.dependencies, packageName);
}
async resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>> {
const output = await this.#listDependencies(packageNames, options);
logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`, { output });
const results: Record<string, string> = {};
for (const packageName of packageNames) {
const version = this.#recursivelySearchDependencies(output.dependencies, packageName);
if (version) {
results[packageName] = version;
}
}
return results;
}
async extractDirectDependenciesMeta(
options: PackageManagerOptions
): Promise<Record<string, DependencyMeta>> {
const result = await this.#listDirectDependencies(options);
logger.debug(`Extracting direct dependencies metadata using ${this.name}`);
return result.dependencies ? this.#flattenDependenciesMeta(result.dependencies) : {};
}
async #listDirectDependencies(options: PackageManagerOptions) {
const childProcess = await $({
cwd: options.cwd,
reject: false,
})`${this.cmd} list --json`;
if (childProcess.failed) {
logger.debug("Failed to list dependencies, using stdout anyway...", {
error: childProcess,
});
}
return JSON.parse(childProcess.stdout) as NpmListOutput;
}
async #listDependencies(packageNames: string[], options: PackageManagerOptions) {
const childProcess = await $({
cwd: options.cwd,
reject: false,
})`${this.cmd} list ${packageNames} --json`;
if (childProcess.failed) {
logger.debug("Failed to list dependencies, using stdout anyway...", {
error: childProcess,
});
}
return JSON.parse(childProcess.stdout) as NpmListOutput;
}
#recursivelySearchDependencies(
dependencies: Record<string, NpmDependency>,
packageName: string
): string | undefined {
for (const [name, dependency] of Object.entries(dependencies)) {
if (name === packageName) {
return dependency.version;
}
if (dependency.dependencies) {
const result = this.#recursivelySearchDependencies(dependency.dependencies, packageName);
if (result) {
return result;
}
}
}
return;
}
#flattenDependenciesMeta(
dependencies: Record<string, NpmDependency>
): Record<string, DependencyMeta> {
let results: Record<string, DependencyMeta> = {};
for (const [name, dep] of Object.entries(dependencies)) {
const { version, resolved, dependencies: children } = dep;
results[name] = { version, external: !!resolved && !resolved.startsWith("file:") };
if (children) {
results = { ...results, ...this.#flattenDependenciesMeta(children) };
}
}
return results;
}
}
class YarnCommands implements PackageManagerCommands {
get name() {
return "yarn";
}
private get cmd() {
return process.platform === "win32" ? "yarn.cmd" : "yarn";
}
async installDependencies(options: PackageManagerOptions) {
const { stdout, stderr } = await $({ cwd: options.cwd })`${this.cmd} install`;
logger.debug(`Installing dependencies using ${this.name}`, { stdout, stderr });
}
async resolveDependencyVersion(packageName: string, options: PackageManagerOptions) {
const { stdout } = await $({ cwd: options.cwd })`${this.cmd} info ${packageName} --json`;
const lines = stdout.split("\n");
logger.debug(`Resolving ${packageName} version using ${this.name}`);
for (const line of lines) {
const json = JSON.parse(line);
if (json.value === packageName) {
return json.children.Version;
}
}
}
async resolveDependencyVersions(
packageNames: string[],
options: PackageManagerOptions
): Promise<Record<string, string>> {
const stdout = await this.#listDependencies(packageNames, options);
const lines = stdout.split("\n");
logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
const results: Record<string, string> = {};
for (const line of lines) {
const json = JSON.parse(line);
const packageName = this.#parseYarnValueIntoPackageName(json.value);
if (packageNames.includes(packageName)) {
results[packageName] = json.children.Version;
}
}
return results;
}
async extractDirectDependenciesMeta(options: PackageManagerOptions) {
const result = await this.#listDirectDependencies(options);
const rawPackagesData = result.split("\n");
logger.debug(`Extracting direct dependencies metadata using ${this.name}`);
const results: Record<string, DependencyMeta> = {};
for (const rawPackageData of rawPackagesData) {
const packageData = JSON.parse(rawPackageData);
const [name, dependencyMeta] = this.#parseYarnValueIntoDependencyMeta(packageData.value);
results[name] = dependencyMeta;
}
return results;
}
async #listDirectDependencies(options: PackageManagerOptions) {
const childProcess = await $({
cwd: options.cwd,
reject: false,
})`${this.cmd} info --all --json`;
if (childProcess.failed) {
logger.debug("Failed to list dependencies, using stdout anyway...", {
error: childProcess,
});
}
return childProcess.stdout;
}
async #listDependencies(packageNames: string[], options: PackageManagerOptions) {
const childProcess = await $({
cwd: options.cwd,
reject: false,
})`${this.cmd} info ${packageNames} --json`;
if (childProcess.failed) {
logger.debug("Failed to list dependencies, using stdout anyway...", {
error: childProcess,
});
}
return childProcess.stdout;
}
// The "value" when doing yarn info is formatted like this:
// "package-name@npm:version" or "package-name@workspace:version"
// This function will parse the value into just the package name.
// This correctly handles scoped packages as well e.g. @scope/package-name@npm:version
#parseYarnValueIntoPackageName(value: string): string {
const parts = value.split("@");
// If the value does not contain an "@" symbol, then it's just the package name
if (parts.length === 3) {
return parts[1] as string;
}
// If the value contains an "@" symbol, then the package name is the first part
return parts[0] as string;
}
#parseYarnValueIntoDependencyMeta(value: string): [string, DependencyMeta] {
const parts = value.split("@");
let name: string, protocol: string, version: string;
if (parts.length === 3) {
// e.g. @<scope>/<package>@<protocol>:<version> -> ["", "<scope>/<package>"", "<protocol>:<version>""]
name = `@${parts[1]}`;
[protocol = "", version = ""] = parts[2]!.split(":");
} else if (parts.length === 2) {
// e.g. <package>@<protocol>:<version> -> ["<package>"", "<protocol>:<version>""]
name = parts[0]!.toString();
[protocol = "", version = ""] = parts[1]!.split(":");
} else {
throw new Error("Failed parsing ${value} into dependency meta");
}
return [
name,
{
version,
external: protocol !== "workspace" && protocol !== "file",
},
];
}
}
function isBuiltInModule(module: string): boolean {
// if the module has node: prefix, it's a built-in module
if (module.startsWith("node:")) {
return true;
}
return builtinModules.includes(module);
}
@@ -1,64 +0,0 @@
import { logger } from "./logger.js";
export async function callResolveEnvVars(
configModule: any,
env: Record<string, string | undefined>,
environment: string,
projectRef: string
): Promise<{ variables: Record<string, string>; override: boolean } | undefined> {
if (
configModule &&
configModule.resolveEnvVars &&
typeof configModule.resolveEnvVars === "function"
) {
let resolvedEnvVars: Record<string, string> = {};
try {
let result = await configModule.resolveEnvVars({
projectRef,
environment,
env,
});
if (!result) {
return;
}
result = await result;
if (typeof result === "object" && result !== null && "variables" in result) {
const variables = result.variables;
if (Array.isArray(variables)) {
for (const item of variables) {
if (
typeof item === "object" &&
item !== null &&
"name" in item &&
"value" in item &&
typeof item.name === "string" &&
typeof item.value === "string"
) {
resolvedEnvVars[item.name] = item.value;
}
}
} else if (typeof variables === "object") {
for (const [key, value] of Object.entries(variables)) {
if (typeof key === "string" && typeof value === "string") {
resolvedEnvVars[key] = value;
}
}
}
}
return {
variables: resolvedEnvVars,
override: result.override,
};
} catch (error) {
logger.error(error);
}
}
return;
}
+23 -4
View File
@@ -1,9 +1,28 @@
import { chalkError } from "./cliOutput.js";
import { logger } from "./logger.js";
export type RuntimeMinimumVersion = {
major: number;
minor: number;
};
const REQUIRED_MINIMUM_VERSIONS: RuntimeMinimumVersion[] = [
{ major: 18, minor: 20 },
{ major: 20, minor: 5 },
];
/**
* This function is used by the dev CLI to make sure that the runtime is compatible
*/
export function runtimeCheck(minimumMajor: number, minimumMinor: number) {
export function runtimeChecks() {
try {
REQUIRED_MINIMUM_VERSIONS.forEach((version) => runtimeCheck(version));
} catch (e) {
logger.log(`${chalkError("X Error:")} ${e}`);
process.exit(1);
}
}
function runtimeCheck(version: RuntimeMinimumVersion) {
// Check if the runtime is Node.js
if (typeof process === "undefined") {
throw "The dev CLI can only be run in a Node.js compatible environment";
@@ -14,11 +33,11 @@ export function runtimeCheck(minimumMajor: number, minimumMinor: number) {
const isBun = typeof process.versions.bun === "string";
if (major < minimumMajor || (major === minimumMajor && minor < minimumMinor)) {
if (major < version.major || (major === version.major && minor < version.minor)) {
if (isBun) {
throw `The dev CLI requires at least Node.js ${minimumMajor}.${minimumMinor}. You are running Bun ${process.versions.bun}, which is compatible with Node.js ${process.versions.node}`;
throw `The dev CLI requires at least Node.js ${version.major}.${version.minor}. You are running Bun ${process.versions.bun}, which is compatible with Node.js ${process.versions.node}`;
} else {
throw `The dev CLI requires at least Node.js ${minimumMajor}.${minimumMinor}. You are running Node.js ${process.versions.node}`;
throw `The dev CLI requires at least Node.js ${version.major}.${version.minor}. You are running Node.js ${process.versions.node}`;
}
}
+7 -1
View File
@@ -98,6 +98,11 @@ export type GetEnvOptions = {
};
export async function getProjectClient(options: GetEnvOptions) {
logger.debug(
`Initializing ${options.env} environment for project ${options.projectRef}`,
options.apiUrl
);
const apiClient = new CliApiClient(options.apiUrl, options.accessToken);
const projectEnv = await apiClient.getProjectEnv({
@@ -112,7 +117,7 @@ export async function getProjectClient(options: GetEnvOptions) {
);
} else {
logger.error(
`Failed to initialize dev environment: ${projectEnv.error}. Using project ref ${options.projectRef}`
`Failed to initialize ${options.env} environment: ${projectEnv.error}. Using project ref ${options.projectRef}`
);
}
@@ -122,6 +127,7 @@ export async function getProjectClient(options: GetEnvOptions) {
const client = new CliApiClient(projectEnv.data.apiUrl, projectEnv.data.apiKey);
return {
id: projectEnv.data.projectId,
name: projectEnv.data.name,
client,
};
@@ -0,0 +1,73 @@
import type {
BackgroundWorkerSourceFileMetadata,
TaskFile,
TaskManifest,
} from "@trigger.dev/core/v3/schemas";
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import * as zlib from "node:zlib";
export async function resolveFileSources(files: TaskFile[], baseDir: string) {
const sources: Record<string, { contents: string; contentHash: string }> = {};
for (const file of files) {
const fullPath = join(baseDir, file.entry);
const content = await readFile(fullPath, "utf-8");
const hasher = createHash("md5");
hasher.update(content);
sources[file.entry] = {
contents: compressContent(content),
contentHash: hasher.digest("hex"),
};
}
return sources;
}
export function resolveTaskSourceFiles(
sources: Record<string, { contents: string; contentHash: string }>,
tasks: TaskManifest[]
): Array<BackgroundWorkerSourceFileMetadata> {
const tasksGroupedByFile: Record<string, TaskManifest[]> = {};
for (const task of tasks) {
if (!tasksGroupedByFile[task.filePath]) {
tasksGroupedByFile[task.filePath] = [];
}
tasksGroupedByFile[task.filePath]!.push(task);
}
const taskFiles: Array<BackgroundWorkerSourceFileMetadata> = [];
for (const [filePath, tasks] of Object.entries(tasksGroupedByFile)) {
const source = sources[filePath];
if (!source) {
continue;
}
const taskIds = tasks.map((task) => task.id);
taskFiles.push({
...source,
taskIds,
filePath,
});
}
return taskFiles;
}
function compressContent(data: string) {
// Convert data to string if it's not already
// Compress the data
const compressedData = zlib.deflateSync(data);
// Encode the compressed data to base64
const base64Encoded = compressedData.toString("base64");
return base64Encoded;
}
+14 -3
View File
@@ -1,8 +1,8 @@
import { z } from "zod";
import { fromZodError } from "zod-validation-error";
import { fromZodError, ValidationError } from "zod-validation-error";
import { RetryOptions } from "../schemas/index.js";
import { calculateNextRetryDelay } from "../utils/retries.js";
import { ApiConnectionError, ApiError } from "./errors.js";
import { ApiConnectionError, ApiError, ApiSchemaValidationError } from "./errors.js";
import { Attributes, Span } from "@opentelemetry/api";
import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
@@ -232,12 +232,23 @@ async function _doZodFetchWithRetries<TResponseBodySchema extends z.ZodTypeAny>(
return { data: parsedResult.data, response };
}
throw fromZodError(parsedResult.error);
const validationError = fromZodError(parsedResult.error);
throw new ApiSchemaValidationError({
status: response.status,
cause: validationError,
message: validationError.message,
rawBody: jsonBody,
headers: responseHeaders,
});
} catch (error) {
if (error instanceof ApiError) {
throw error;
}
if (error instanceof ValidationError) {
}
if (options?.retry) {
const retry = { ...defaultRetryOptions, ...options.retry };
+26
View File
@@ -155,6 +155,32 @@ export class RateLimitError extends ApiError {
export class InternalServerError extends ApiError {}
export class ApiSchemaValidationError extends ApiError {
override readonly status: 200 = 200;
readonly rawBody: any;
constructor({
message,
cause,
status,
rawBody,
headers,
}: {
message?: string;
cause?: Error | undefined;
status: number;
rawBody: any;
headers: APIHeaders | undefined;
}) {
super(status, undefined, message || "Validation error.", headers);
// in some environments the 'cause' property is already declared
// @ts-ignore
if (cause) this.cause = cause;
this.rawBody = rawBody;
}
}
function castToError(err: any): Error {
if (err instanceof Error) return err;
return new Error(err);
+8
View File
@@ -24,10 +24,17 @@ export interface BuildExtension {
) => Promise<undefined | void> | undefined | void;
}
export interface BuildLogger {
debug: (...args: unknown[]) => void;
log: (...args: unknown[]) => void;
warn: (...args: unknown[]) => void;
}
export interface BuildContext {
target: BuildTarget;
config: ResolvedConfig;
workingDir: string;
logger: BuildLogger;
addLayer(layer: BuildLayer): void;
registerPlugin(plugin: Plugin, options?: RegisterPluginOptions): void;
@@ -47,6 +54,7 @@ export interface BuildLayer {
};
deploy?: {
env?: Record<string, string | undefined>;
override?: boolean;
};
dependencies?: Record<string, string>;
}
+3 -1
View File
@@ -2,6 +2,7 @@ import { type Defu } from "defu";
import type { Prettify } from "ts-essentials";
import { TriggerConfig } from "../config.js";
import { BuildRuntime } from "../schemas/build.js";
import { ResolveEnvironmentVariablesFunction } from "../types/index.js";
export type ResolvedConfig = Prettify<
Defu<
@@ -23,5 +24,6 @@ export type ResolvedConfig = Prettify<
packageJsonPath: string;
lockfilePath: string;
configFile?: string;
resolveEnvVars?: ResolveEnvironmentVariablesFunction;
}
>;
>;
+4 -4
View File
@@ -41,7 +41,7 @@ export class PrismaExtension implements BuildExtension {
// Resolve the path to the prisma schema, relative to the config.directory
this._resolvedSchemaPath = resolve(context.workingDir, this.options.schema);
console.log(`Resolved the prisma schema to: ${this._resolvedSchemaPath}`);
context.logger.debug(`Resolved the prisma schema to: ${this._resolvedSchemaPath}`);
// Check that the prisma schema exists
if (!existsSync(this._resolvedSchemaPath)) {
@@ -58,7 +58,7 @@ export class PrismaExtension implements BuildExtension {
assert(this._resolvedSchemaPath, "Resolved schema path is not set");
console.log("Looking for @prisma/client in the externals", {
context.logger.debug("Looking for @prisma/client in the externals", {
externals: manifest.externals,
});
@@ -74,7 +74,7 @@ export class PrismaExtension implements BuildExtension {
);
}
console.log(`PrismaExtension is generating the Prisma client for version ${version}`);
context.logger.debug(`PrismaExtension is generating the Prisma client for version ${version}`);
// Now we need to add a layer that:
// Copies the prisma schema to the build outputPath
@@ -82,7 +82,7 @@ export class PrismaExtension implements BuildExtension {
// Adds the `prisma generate` command, which generates the Prisma client
const schemaDestinationPath = join(manifest.outputPath, "prisma", "schema.prisma");
// Copy the prisma schema to the build output path
console.log(
context.logger.debug(
`Copying the prisma schema from ${this._resolvedSchemaPath} to ${schemaDestinationPath}`
);
@@ -0,0 +1,156 @@
import { BuildExtension } from "../build/extensions.js";
export type SyncEnvVarsBody = Record<string, string> | Array<{ name: string; value: string }>;
export type SyncEnvVarsResult =
| SyncEnvVarsBody
| Promise<void | undefined | SyncEnvVarsBody>
| void
| undefined;
export type SyncEnvVarsParams = {
projectRef: string;
environment: string;
env: Record<string, string>;
};
const UNSYNCABLE_ENV_VARS = [
"PWD",
"MallocNanoZone",
"USER",
"LANG",
"__CFBundleIdentifier",
"COMMAND_MODE",
"PATH",
"LOGNAME",
"SSLKEYLOGFILE",
"SSH_AUTH_SOCK",
"SHLVL",
"SHELL",
"HOME",
"__CF_USER_TEXT_ENCODING",
"XPC_SERVICE_NAME",
"XPC_FLAGS",
"ORIGINAL_XDG_CURRENT_DESKTOP",
"TERM_PROGRAM",
"TERM_PROGRAM_VERSION",
"COLORTERM",
"GIT_ASKPASS",
"VSCODE_GIT_ASKPASS_NODE",
"VSCODE_GIT_ASKPASS_EXTRA_ARGS",
"VSCODE_GIT_ASKPASS_MAIN",
"VSCODE_GIT_IPC_HANDLE",
"VSCODE_INJECTION",
"ZDOTDIR",
"USER_ZDOTDIR",
"TERM",
"OLDPWD",
"HOMEBREW_PREFIX",
"HOMEBREW_CELLAR",
"HOMEBREW_REPOSITORY",
"MANPATH",
"INFOPATH",
"__GIT_PROMPT_DIR",
"GIT_PROMPT_EXECUTABLE",
"NVM_DIR",
"NVM_CD_FLAGS",
"NVM_BIN",
"NVM_INC",
"BUN_INSTALL",
"DENO_INSTALL",
"GITHUB_TOKEN",
"TMPDIR",
"_",
];
export type SyncEnvVarsFunction = (params: SyncEnvVarsParams) => SyncEnvVarsResult;
export type SyncEnvVarsOptions = {
override?: boolean;
};
export function syncEnvVars(fn: SyncEnvVarsFunction, options?: SyncEnvVarsOptions): BuildExtension {
return {
name: "SyncEnvVarsExtension",
async onBuildComplete(context, manifest) {
if (context.target === "dev") {
return;
}
const result = await callSyncEnvVarsFn(
fn,
manifest.deploy.env ?? {},
manifest.environment,
context.config.project
);
if (!result) {
return;
}
const env = Object.entries(result).reduce(
(acc, [key, value]) => {
if (UNSYNCABLE_ENV_VARS.includes(key)) {
return acc;
}
acc[key] = value;
return acc;
},
{} as Record<string, string>
);
context.addLayer({
id: "sync-env-vars",
deploy: {
env,
override: options?.override ?? true,
},
});
},
};
}
async function callSyncEnvVarsFn(
syncEnvVarsFn: SyncEnvVarsFunction | undefined,
env: Record<string, string>,
environment: string,
projectRef: string
): Promise<Record<string, string> | undefined> {
if (syncEnvVarsFn && typeof syncEnvVarsFn === "function") {
let resolvedEnvVars: Record<string, string> = {};
let result = syncEnvVarsFn({
projectRef,
environment,
env,
});
if (!result) {
return;
}
result = await result;
if (Array.isArray(result)) {
for (const item of result) {
if (
typeof item === "object" &&
item !== null &&
"name" in item &&
"value" in item &&
typeof item.name === "string" &&
typeof item.value === "string"
) {
resolvedEnvVars[item.name] = item.value;
}
}
} else if (result) {
resolvedEnvVars = result;
}
return resolvedEnvVars;
}
return;
}
+4
View File
@@ -57,3 +57,7 @@ export {
} from "./utils/ioSerialization.js";
export * from "./config.js";
import { VERSION } from "../version.js";
export { VERSION as CORE_VERSION };
+22 -2
View File
@@ -35,6 +35,7 @@ export const GetProjectEnvResponse = z.object({
apiKey: z.string(),
name: z.string(),
apiUrl: z.string(),
projectId: z.string(),
});
export type GetProjectEnvResponse = z.infer<typeof GetProjectEnvResponse>;
@@ -153,6 +154,13 @@ export type StartDeploymentIndexingResponseBody = z.infer<
typeof StartDeploymentIndexingResponseBody
>;
export const FinalizeDeploymentRequestBody = z.object({
imageReference: z.string(),
selfHosted: z.boolean().optional(),
});
export type FinalizeDeploymentRequestBody = z.infer<typeof FinalizeDeploymentRequestBody>;
export const ExternalBuildData = z.object({
buildId: z.string(),
buildToken: z.string(),
@@ -187,6 +195,18 @@ export const DeploymentErrorData = z.object({
stderr: z.string().optional(),
});
export const FailDeploymentRequestBody = z.object({
error: DeploymentErrorData,
});
export type FailDeploymentRequestBody = z.infer<typeof FailDeploymentRequestBody>;
export const FailDeploymentResponseBody = z.object({
id: z.string(),
});
export type FailDeploymentResponseBody = z.infer<typeof FailDeploymentResponseBody>;
export const GetDeploymentResponseBody = z.object({
id: z.string(),
status: z.enum([
@@ -201,8 +221,8 @@ export const GetDeploymentResponseBody = z.object({
contentHash: z.string(),
shortCode: z.string(),
version: z.string(),
imageReference: z.string().optional(),
errorData: DeploymentErrorData.optional().nullable(),
imageReference: z.string().nullish(),
errorData: DeploymentErrorData.nullish(),
worker: z
.object({
id: z.string(),
+10
View File
@@ -19,10 +19,19 @@ export type BuildRuntime = z.infer<typeof BuildRuntime>;
export const BuildManifest = z.object({
target: BuildTarget,
packageVersion: z.string(),
cliPackageVersion: z.string(),
contentHash: z.string(),
runtime: BuildRuntime,
environment: z.string(),
config: ConfigManifest,
files: z.array(TaskFile),
sources: z.record(
z.object({
contents: z.string(),
contentHash: z.string(),
})
),
outputPath: z.string(),
indexerEntryPoint: z.string(),
workerEntryPoint: z.string(),
@@ -35,6 +44,7 @@ export const BuildManifest = z.object({
}),
deploy: z.object({
env: z.record(z.string()).optional(),
needsSyncing: z.boolean().optional(),
}),
});
+3 -3
View File
@@ -15,21 +15,21 @@ export const TaskResource = z.object({
export type TaskResource = z.infer<typeof TaskResource>;
export const BackgroundWorkerFileMetadata = z.object({
export const BackgroundWorkerSourceFileMetadata = z.object({
filePath: z.string(),
contents: z.string(),
contentHash: z.string(),
taskIds: z.array(z.string()),
});
export type BackgroundWorkerFileMetadata = z.infer<typeof BackgroundWorkerFileMetadata>;
export type BackgroundWorkerSourceFileMetadata = z.infer<typeof BackgroundWorkerSourceFileMetadata>;
export const BackgroundWorkerMetadata = z.object({
packageVersion: z.string(),
contentHash: z.string(),
cliPackageVersion: z.string().optional(),
tasks: z.array(TaskResource),
fileContents: z.array(BackgroundWorkerFileMetadata).optional(),
sourceFiles: z.array(BackgroundWorkerSourceFileMetadata).optional(),
});
export type BackgroundWorkerMetadata = z.infer<typeof BackgroundWorkerMetadata>;
+1 -1
View File
@@ -78,7 +78,7 @@ export type ResolveEnvironmentVariablesResult =
export type ResolveEnvironmentVariablesParams = {
projectRef: string;
environment: "dev" | "staging" | "prod";
environment: string;
env: Record<string, string>;
};
+93 -447
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -2,6 +2,7 @@
"name": "@references/v3-catalog",
"version": "0.1.0",
"private": true,
"type": "module",
"zenstack": {
"schema": "./prisma/schema.zmodel"
},
+7 -1
View File
@@ -1,7 +1,13 @@
import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
import { defineConfig } from "@trigger.dev/sdk/v3";
import { defineConfig, ResolveEnvironmentVariablesFunction } from "@trigger.dev/sdk/v3";
import { emitDecoratorMetadata } from "@trigger.dev/sdk/v3/extensions";
export const resolveEnvVars: ResolveEnvironmentVariablesFunction = async ({
projectRef,
env,
environment,
}) => {};
export default defineConfig({
project: "yubjwjsfkxnylobaqvqz",
machine: "small-2x",