diff --git a/packages/cli-v3/src/build/bundle.ts b/packages/cli-v3/src/build/bundle.ts index fcaae5bf4..8a01c8422 100644 --- a/packages/cli-v3/src/build/bundle.ts +++ b/packages/cli-v3/src/build/bundle.ts @@ -14,6 +14,8 @@ import { shims, } from "./packageModules.js"; import { buildPlugins } from "./plugins.js"; +import { createHash, hash } from "node:crypto"; +import { createFile } from "../utilities/fileSystem.js"; export interface BundleOptions { target: BuildTarget; @@ -28,6 +30,7 @@ export interface BundleOptions { } export type BundleResult = { + contentHash: string; files: TaskFile[]; configPath: string; loaderEntryPoint: string | undefined; @@ -62,7 +65,7 @@ export async function bundleWorker(options: BundleOptions): Promise -): Omit | undefined { + result: esbuild.BuildResult<{ metafile: true; write: false }> +): Promise | undefined> { + const hasher = createHash("md5"); + + for (const outputFile of result.outputFiles) { + hasher.update(outputFile.hash); + + await createFile(outputFile.path, outputFile.contents); + } + const files: Array<{ entry: string; out: string }> = []; let configPath: string | undefined; @@ -149,10 +160,15 @@ export function getBundleResultFromBuild( } else if (isEntryPointForTarget(outputMeta.entryPoint, target)) { workerEntryPoint = $outputPath; } else { - files.push({ - entry: outputMeta.entryPoint, - out: $outputPath, - }); + if ( + !outputMeta.entryPoint.startsWith("..") && + !outputMeta.entryPoint.includes("node_modules") + ) { + files.push({ + entry: outputMeta.entryPoint, + out: $outputPath, + }); + } } } } @@ -166,6 +182,7 @@ export function getBundleResultFromBuild( configPath: configPath, loaderEntryPoint, workerEntryPoint, + contentHash: hasher.digest("hex"), }; } diff --git a/packages/cli-v3/src/build/externals.ts b/packages/cli-v3/src/build/externals.ts index 904b38497..7044c9dae 100644 --- a/packages/cli-v3/src/build/externals.ts +++ b/packages/cli-v3/src/build/externals.ts @@ -1,4 +1,3 @@ - import * as esbuild from "esbuild"; import { makeRe } from "minimatch"; import { mkdir, symlink } from "node:fs/promises"; @@ -20,10 +19,7 @@ const FORCED_EXTERNALS = ["import-in-the-middle"]; * This function will create a symbolic link from a place where the external is resolvable * to the actual resolved external path */ -async function linkUnresolvableExternals( - externals: Array, - resolveDir: string -) { +async function linkUnresolvableExternals(externals: Array, resolveDir: string) { for (const external of externals) { if (!(await isExternalResolvable(external, resolveDir))) { logger.debug("External is not resolvable", { external }); @@ -44,10 +40,7 @@ async function linkExternal(external: CollectedExternal, resolveDir: string) { await symlink(external.path, join(destinationPath, external.name), "dir"); } -async function isExternalResolvable( - external: CollectedExternal, - resolveDir: string -) { +async function isExternalResolvable(external: CollectedExternal, resolveDir: string) { try { const resolvedPath = nodeResolve.sync(external.name, { basedir: resolveDir, @@ -60,7 +53,6 @@ async function isExternalResolvable( return true; } catch (e) { - console.error("Could not resolve external", { external, resolveDir, e }); return false; } } @@ -100,70 +92,65 @@ function createExternalsCollector( }); maybeExternals.forEach((external) => { - build.onResolve( - { filter: external.filter, namespace: "file" }, - async (args) => { - const resolvedPath = nodeResolve.sync(args.path, { - basedir: args.resolveDir, - }); + build.onResolve({ filter: external.filter, namespace: "file" }, async (args) => { + const resolvedPath = nodeResolve.sync(args.path, { + basedir: args.resolveDir, + }); - logger.debug("Resolved external", { - external, - resolvedPath, - args, - }); + logger.debug("Resolved external", { + external, + resolvedPath, + args, + }); - const packageJsonPath = await resolvePackageJSON( - dirname(resolvedPath) - ); + const packageJsonPath = await resolvePackageJSON(dirname(resolvedPath)); - if (!packageJsonPath) { - return undefined; - } - - logger.debug("Found package.json", { packageJsonPath }); - - const packageJson = await readPackageJSON(packageJsonPath); - - if (!packageJson || !packageJson.name) { - return undefined; - } - - if (!external.filter.test(packageJson.name)) { - logger.debug("Package name does not match", { - external, - packageJson, - }); - - return undefined; - } - - if (!packageJson.version) { - logger.debug("No version found in package.json", { - external, - packageJson, - }); - - return undefined; - } - - externals.push({ - name: packageJson.name, - path: dirname(packageJsonPath), - version: packageJson.version, - }); - - logger.debug("Resolved external", { - external, - resolvedPath, - args, - }); - - return { - external: true, - }; + if (!packageJsonPath) { + return undefined; } - ); + + logger.debug("Found package.json", { packageJsonPath }); + + const packageJson = await readPackageJSON(packageJsonPath); + + if (!packageJson || !packageJson.name) { + return undefined; + } + + if (!external.filter.test(packageJson.name)) { + logger.debug("Package name does not match", { + external, + packageJson, + }); + + return undefined; + } + + if (!packageJson.version) { + logger.debug("No version found in package.json", { + external, + packageJson, + }); + + return undefined; + } + + externals.push({ + name: packageJson.name, + path: dirname(packageJsonPath), + version: packageJson.version, + }); + + logger.debug("Resolved external", { + external, + resolvedPath, + args, + }); + + return { + external: true, + }; + }); }); }, }, @@ -172,10 +159,7 @@ function createExternalsCollector( type MaybeExternal = { raw: string; filter: RegExp }; -function discoverMaybeExternals( - target: BuildTarget, - config: ResolvedConfig -): Array { +function discoverMaybeExternals(target: BuildTarget, config: ResolvedConfig): Array { const external: Array = []; for (const externalName of FORCED_EXTERNALS) { @@ -187,9 +171,7 @@ function discoverMaybeExternals( external.push({ raw: externalName, - filter: new RegExp( - `^${externalName}$|${externalRegex.source}` - ), + filter: new RegExp(`^${externalName}$|${externalRegex.source}`), }); } @@ -212,14 +194,12 @@ function discoverMaybeExternals( const externalRegex = makeRe(externalName); if (!externalRegex) { - continue + continue; } external.push({ raw: externalName, - filter: new RegExp( - `^${externalName}$|${externalRegex.source}` - ), + filter: new RegExp(`^${externalName}$|${externalRegex.source}`), }); } @@ -235,9 +215,7 @@ function discoverMaybeExternals( external.push({ raw: externalName, - filter: new RegExp( - `^${externalName}$|${externalRegex.source}` - ), + filter: new RegExp(`^${externalName}$|${externalRegex.source}`), }); } } diff --git a/packages/cli-v3/src/dev/backgroundWorker.ts b/packages/cli-v3/src/dev/backgroundWorker.ts index ad6652f12..c61a3911c 100644 --- a/packages/cli-v3/src/dev/backgroundWorker.ts +++ b/packages/cli-v3/src/dev/backgroundWorker.ts @@ -115,9 +115,13 @@ export class BackgroundWorkerCoordinator { await worker.cancelRun(taskRunId); } - async registerWorker(record: CreateBackgroundWorkerResponse, worker: BackgroundWorker) { + async registerWorker(worker: BackgroundWorker) { + if (!worker.serverWorker) { + return; + } + for (const [workerId, existingWorker] of this._backgroundWorkers.entries()) { - if (workerId === record.id) { + if (workerId === worker.serverWorker.id) { continue; } @@ -125,11 +129,19 @@ export class BackgroundWorkerCoordinator { this.onWorkerDeprecated.post({ worker: existingWorker, id: workerId }); } - this._backgroundWorkers.set(record.id, worker); - this.onWorkerRegistered.post({ worker, id: record.id, record }); + this._backgroundWorkers.set(worker.serverWorker.id, worker); + this.onWorkerRegistered.post({ + worker, + id: worker.serverWorker.id, + record: worker.serverWorker, + }); worker.onTaskRunHeartbeat.attach((id) => { - this.onWorkerTaskRunHeartbeat.post({ id, backgroundWorkerId: record.id, worker }); + this.onWorkerTaskRunHeartbeat.post({ + id, + backgroundWorkerId: worker.serverWorker!.id, + worker, + }); }); } diff --git a/packages/cli-v3/src/dev/devSession.ts b/packages/cli-v3/src/dev/devSession.ts index 183b42578..a81b7bcf9 100644 --- a/packages/cli-v3/src/dev/devSession.ts +++ b/packages/cli-v3/src/dev/devSession.ts @@ -22,6 +22,7 @@ import { logger } from "../utilities/logger.js"; import { EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js"; import { copyManifestToDir } from "../build/manifests.js"; import { startWorkerRuntime } from "./workerRuntime.js"; +import { chalkGrey } from "../utilities/cliOutput.js"; export type DevSessionOptions = { name: string | undefined; @@ -78,7 +79,7 @@ export async function startDevSession({ } async function updateBuild(build: esbuild.BuildResult, workerDir: EphemeralDirectory) { - const bundle = getBundleResultFromBuild("dev", rawConfig.workingDir, build); + const bundle = await getBundleResultFromBuild("dev", rawConfig.workingDir, build); if (bundle) { await updateBundle({ ...bundle, stop: undefined }, workerDir); @@ -91,6 +92,10 @@ export async function startDevSession({ setup(b: esbuild.PluginBuild) { b.onStart(() => { logger.debug("on-end plugin started"); + + if (bundled) { + logger.log(chalkGrey("○ Rebuilding background worker…")); + } }); b.onEnd(async (result: esbuild.BuildResult) => { const errors = result.errors; @@ -108,7 +113,7 @@ export async function startDevSession({ // First bundle, no need to update bundle bundled = true; } else { - const workerDir = getTmpDir(rawConfig.workingDir, "worker"); + const workerDir = getTmpDir(rawConfig.workingDir, "build"); await updateBuild(result, workerDir); } @@ -129,6 +134,8 @@ export async function startDevSession({ jsxAutomatic: rawConfig.build.jsx.automatic, }); + logger.log(chalkGrey("○ Building background worker…")); + await updateBundle(bundleResult); return bundleResult.stop; @@ -154,6 +161,7 @@ async function createBuildManifestFromBundle( workerDir: string | undefined ): Promise { const buildManifest: BuildManifest = { + contentHash: bundle.contentHash, runtime: resolvedConfig.runtime ?? DEFAULT_RUNTIME, target: "dev", files: bundle.files, diff --git a/packages/cli-v3/src/dev/workerRuntime.ts b/packages/cli-v3/src/dev/workerRuntime.ts index 46a5d9514..50ae653d1 100644 --- a/packages/cli-v3/src/dev/workerRuntime.ts +++ b/packages/cli-v3/src/dev/workerRuntime.ts @@ -1,9 +1,12 @@ import { BuildManifest, clientWebsocketMessages, + CreateBackgroundWorkerRequestBody, SemanticInternalAttributes, serverWebsocketMessages, + TaskManifest, TaskRunExecutionLazyAttemptPayload, + WorkerManifest, } from "@trigger.dev/core/v3"; import { ResolvedConfig } from "@trigger.dev/core/v3/build"; import { ClientRequestArgs } from "node:http"; @@ -11,7 +14,7 @@ import { WebSocket } from "partysocket"; import { ClientOptions, WebSocket as wsWebSocket } from "ws"; import { CliApiClient } from "../apiClient.js"; import { DevCommandOptions } from "../commands/dev.js"; -import { chalkError } from "../utilities/cliOutput.js"; +import { chalkError, chalkGrey, chalkTask } from "../utilities/cliOutput.js"; import { logger } from "../utilities/logger.js"; import { BackgroundWorker, BackgroundWorkerCoordinator } from "./backgroundWorker.js"; import { @@ -20,6 +23,7 @@ import { ZodMessageSender, } from "@trigger.dev/core/v3/zodMessageHandler"; import { resolveDotEnvVars } from "../utilities/dotEnv.js"; +import { VERSION } from "../version.js"; export interface WorkerRuntime { shutdown(): Promise; @@ -47,6 +51,7 @@ class DevWorkerRuntime implements WorkerRuntime { private backgroundWorkerCoordinator: BackgroundWorkerCoordinator; private sender: ZodMessageSender; private websocketMessageHandler: ZodMessageHandler; + private lastBuild: BuildManifest | undefined; constructor(public readonly options: WorkerRuntimeOptions) { const websocketUrl = new URL(this.options.client.apiURL); @@ -151,6 +156,11 @@ class DevWorkerRuntime implements WorkerRuntime { } async initializeWorker(manifest: BuildManifest, options?: { cwd?: string }): Promise { + if (this.lastBuild && this.lastBuild.contentHash === manifest.contentHash) { + logger.log(chalkGrey("○ No changes detected, skipping build…")); + return; + } + const env = await this.#getEnvVars(); const backgroundWorker = new BackgroundWorker(manifest, { @@ -159,6 +169,41 @@ class DevWorkerRuntime implements WorkerRuntime { }); await backgroundWorker.initialize(); + + if (!backgroundWorker.manifest) { + throw new Error("Could not initialize worker"); + } + + const issues = validateWorkerManifest(backgroundWorker.manifest); + + if (issues.length > 0) { + issues.forEach((issue) => logger.error(issue)); + return; + } + + const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = { + localOnly: true, + metadata: { + packageVersion: VERSION, + cliPackageVersion: VERSION, + tasks: backgroundWorker.manifest.tasks, + contentHash: manifest.contentHash, + }, + supportsLazyAttempts: true, + }; + + const backgroundWorkerRecord = await this.options.client.createBackgroundWorker( + this.options.config.project, + backgroundWorkerBody + ); + + if (!backgroundWorkerRecord.success) { + throw new Error(backgroundWorkerRecord.error); + } + + backgroundWorker.serverWorker = backgroundWorkerRecord.data; + this.backgroundWorkerCoordinator.registerWorker(backgroundWorker); + this.lastBuild = manifest; } async #getEnvVars(): Promise> { @@ -270,3 +315,38 @@ function gatherProcessEnv() { // Filter out undefined values return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined)); } + +function validateWorkerManifest(manifest: WorkerManifest): string[] { + const issues: string[] = []; + + if (!manifest.tasks || manifest.tasks.length === 0) { + issues.push("No tasks defined. Make sure you are exporting tasks."); + } + + // Check for any duplicate task ids + const taskIds = manifest.tasks.map((task) => task.id); + const duplicateTaskIds = taskIds.filter((id, index) => taskIds.indexOf(id) !== index); + + if (duplicateTaskIds.length > 0) { + issues.push(createDuplicateTaskIdOutputErrorMessage(duplicateTaskIds, manifest.tasks)); + } + + return issues; +} + +function createDuplicateTaskIdOutputErrorMessage( + duplicateTaskIds: Array, + tasks: Array +) { + const duplicateTable = duplicateTaskIds + .map((id) => { + const $tasks = tasks.filter((task) => task.id === id); + + return `\n\n${chalkTask(id)} was found in:${tasks + .map((task) => `\n${task.filePath} -> ${task.exportName}`) + .join("")}`; + }) + .join(""); + + return `Duplicate ${chalkTask("task id")} detected:${duplicateTable}`; +} diff --git a/packages/cli-v3/src/entryPoints/dev.ts b/packages/cli-v3/src/entryPoints/dev.ts index ad9b6dbd6..e8b5a29e4 100644 --- a/packages/cli-v3/src/entryPoints/dev.ts +++ b/packages/cli-v3/src/entryPoints/dev.ts @@ -120,8 +120,7 @@ async function registerTaskFileMetadata(files: Array<{ entry: string; out: strin if (taskCatalog.taskExists(task.id)) { taskCatalog.registerTaskFileMetadata(task.id, { exportName, - filePath: file.out, - entryPoint: file.entry, + filePath: file.entry, }); } } diff --git a/packages/cli-v3/src/utilities/fileSystem.ts b/packages/cli-v3/src/utilities/fileSystem.ts index 690540b18..f29522d25 100644 --- a/packages/cli-v3/src/utilities/fileSystem.ts +++ b/packages/cli-v3/src/utilities/fileSystem.ts @@ -5,7 +5,10 @@ import { tmpdir } from "node:os"; import pathModule from "node:path"; // Creates a file at the given path, if the directory doesn't exist it will be created -export async function createFile(path: string, contents: string): Promise { +export async function createFile( + path: string, + contents: string | NodeJS.ArrayBufferView +): Promise { await fsModule.mkdir(pathModule.dirname(path), { recursive: true }); await fsModule.writeFile(path, contents); diff --git a/packages/core/src/v3/extensions/emitDecoratorMetadata.ts b/packages/core/src/v3/extensions/emitDecoratorMetadata.ts index 17e2339b2..dc80f556c 100644 --- a/packages/core/src/v3/extensions/emitDecoratorMetadata.ts +++ b/packages/core/src/v3/extensions/emitDecoratorMetadata.ts @@ -34,10 +34,6 @@ function plugin(options: EmitDecoratorMetadataOptions = {}): esbuild.Plugin { return; } - console.log("Setting up typescript decorators plugin", { - tsconfig, - }); - build.onLoad({ filter: /\.ts$/ }, async (args) => { const ts = await readFile(args.path, "utf8"); diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index ee3cd1916..3f9d97816 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -19,6 +19,7 @@ export type BuildRuntime = z.infer; export const BuildManifest = z.object({ target: BuildTarget, + contentHash: z.string(), runtime: BuildRuntime, config: ConfigManifest, files: z.array(TaskFile), diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index 2da208b9d..e4d37a919 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -169,7 +169,6 @@ export type TaskFile = z.infer; const taskFileMetadata = { filePath: z.string(), - entryPoint: z.string(), exportName: z.string(), };