feat(cli): implements content-addressable store for the dev CLI build outputs, reducing disk usage (#2725)

* feat(cli): implements content-addressable store for the dev CLI build outputs, reducing disk usage

* fix a few things

.changeset/polite-eels-divide.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+feat(cli): implements content-addressable store for the dev CLI build outputs, reducing disk usage

packages/cli-v3/src/build/bundle.ts

@@ -3,8 +3,8 @@ import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build";
 import { BuildManifest, BuildTarget, TaskFile } from "@trigger.dev/core/v3/schemas";
 import * as esbuild from "esbuild";
 import { createHash } from "node:crypto";
-import { join, relative, resolve } from "node:path";
-import { createFile } from "../utilities/fileSystem.js";
+import { basename, join, relative, resolve } from "node:path";
+import { createFile, createFileWithStore } from "../utilities/fileSystem.js";
 import { logger } from "../utilities/logger.js";
 import { resolveFileSources } from "../utilities/sourceFiles.js";
 import { VERSION } from "../version.js";
@@ -37,6 +37,8 @@ export interface BundleOptions {
   jsxAutomatic?: boolean;
   watch?: boolean;
   plugins?: esbuild.Plugin[];
+  /** Shared store directory for deduplicating chunk files via hardlinks */
+  storeDir?: string;
 }
 
 export type BundleResult = {
@@ -51,6 +53,8 @@ export type BundleResult = {
   indexControllerEntryPoint: string | undefined;
   initEntryPoint: string | undefined;
   stop: (() => Promise<void>) | undefined;
+  /** Maps output file paths to their content hashes for deduplication */
+  outputHashes: Record<string, string>;
 };
 
 export class BundleError extends Error {
@@ -159,7 +163,8 @@ export async function bundleWorker(options: BundleOptions): Promise<BundleResult
     options.target,
     options.cwd,
     options.resolvedConfig,
-    result
+    result,
+    options.storeDir
   );
 
   if (!bundleResult) {
@@ -233,14 +238,23 @@ export async function getBundleResultFromBuild(
   target: BuildTarget,
   workingDir: string,
   resolvedConfig: ResolvedConfig,
-  result: esbuild.BuildResult<{ metafile: true; write: false }>
+  result: esbuild.BuildResult<{ metafile: true; write: false }>,
+  storeDir?: string
 ): Promise<Omit<BundleResult, "stop"> | undefined> {
   const hasher = createHash("md5");
+  const outputHashes: Record<string, string> = {};
 
   for (const outputFile of result.outputFiles) {
     hasher.update(outputFile.hash);
+    // Store the hash for each output file (keyed by path)
+    outputHashes[outputFile.path] = outputFile.hash;
 
-    await createFile(outputFile.path, outputFile.contents);
+    if (storeDir) {
+      // Use content-addressable store with esbuild's built-in hash for ALL files
+      await createFileWithStore(outputFile.path, outputFile.contents, storeDir, outputFile.hash);
+    } else {
+      await createFile(outputFile.path, outputFile.contents);
+    }
   }
 
   const files: Array<{ entry: string; out: string }> = [];
@@ -308,6 +322,7 @@ export async function getBundleResultFromBuild(
     initEntryPoint,
     contentHash: hasher.digest("hex"),
     metafile: result.metafile,
+    outputHashes,
   };
 }
 
@@ -354,6 +369,7 @@ export async function createBuildManifestFromBundle({
   target,
   envVars,
   sdkVersion,
+  storeDir,
 }: {
   bundle: BundleResult;
   destination: string;
@@ -364,6 +380,7 @@ export async function createBuildManifestFromBundle({
   target: BuildTarget;
   envVars?: Record<string, string>;
   sdkVersion?: string;
+  storeDir?: string;
 }): Promise<BuildManifest> {
   const buildManifest: BuildManifest = {
     contentHash: bundle.contentHash,
@@ -397,11 +414,12 @@ export async function createBuildManifestFromBundle({
     otelImportHook: {
       include: resolvedConfig.instrumentedPackageNames ?? [],
     },
+    outputHashes: bundle.outputHashes,
   };
 
   if (!workerDir) {
     return buildManifest;
   }
 
-  return copyManifestToDir(buildManifest, destination, workerDir);
+  return copyManifestToDir(buildManifest, destination, workerDir, storeDir);
 }

packages/cli-v3/src/build/manifests.ts

@@ -1,16 +1,26 @@
 import { BuildManifest } from "@trigger.dev/core/v3/schemas";
-import { cp } from "node:fs/promises";
+import { cp, link, mkdir, readdir, readFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
 import { logger } from "../utilities/logger.js";
+import { sanitizeHashForFilename } from "../utilities/fileSystem.js";
 
 export async function copyManifestToDir(
   manifest: BuildManifest,
   source: string,
-  destination: string
+  destination: string,
+  storeDir?: string
 ): Promise<BuildManifest> {
-  // Copy the dir in destination to workerDir
-  await cp(source, destination, { recursive: true });
+  // Copy the dir from source to destination
+  // If storeDir is provided, create hardlinks for files that exist in the store
+  if (storeDir) {
+    await copyDirWithStore(source, destination, storeDir, manifest.outputHashes);
+  } else {
+    await cp(source, destination, { recursive: true });
+  }
 
-  logger.debug("Copied manifest to dir", { source, destination });
+  logger.debug("Copied manifest to dir", { source, destination, storeDir });
 
   // Then update the manifest to point to the new workerDir
   const updatedManifest = { ...manifest };
@@ -37,3 +47,68 @@ export async function copyManifestToDir(
 
   return updatedManifest;
 }
+
+/**
+ * Computes a hash of file contents to use as content-addressable key.
+ * This is a fallback for when outputHashes is not available.
+ */
+async function computeFileHash(filePath: string): Promise<string> {
+  const contents = await readFile(filePath);
+  return createHash("sha256").update(contents).digest("hex").slice(0, 16);
+}
+
+/**
+ * Recursively copies a directory, using hardlinks for files that exist in the store.
+ * This preserves disk space savings from the content-addressable store.
+ *
+ * @param source - Source directory path
+ * @param destination - Destination directory path
+ * @param storeDir - Content-addressable store directory
+ * @param outputHashes - Optional map of file paths to their content hashes (from BuildManifest)
+ */
+async function copyDirWithStore(
+  source: string,
+  destination: string,
+  storeDir: string,
+  outputHashes?: Record<string, string>
+): Promise<void> {
+  await mkdir(destination, { recursive: true });
+
+  const entries = await readdir(source, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const sourcePath = join(source, entry.name);
+    const destPath = join(destination, entry.name);
+
+    if (entry.isDirectory()) {
+      // Recursively copy subdirectories
+      await copyDirWithStore(sourcePath, destPath, storeDir, outputHashes);
+    } else if (entry.isFile()) {
+      // Try to get hash from manifest first, otherwise compute it
+      const contentHash = outputHashes?.[sourcePath] ?? (await computeFileHash(sourcePath));
+      // Sanitize hash to be filesystem-safe (base64 can contain / and +)
+      const safeHash = sanitizeHashForFilename(contentHash);
+      const storePath = join(storeDir, safeHash);
+
+      if (existsSync(storePath)) {
+        // Create hardlink to store file
+        // Fall back to copy if hardlink fails (e.g., on Windows or cross-device)
+        try {
+          await link(storePath, destPath);
+        } catch (linkError) {
+          try {
+            await cp(storePath, destPath);
+          } catch (copyError) {
+            throw linkError; // Rethrow original error if copy also fails
+          }
+        }
+      } else {
+        // File wasn't in the store - copy normally
+        await cp(sourcePath, destPath);
+      }
+    } else if (entry.isSymbolicLink()) {
+      // Preserve symbolic links (e.g., node_modules links)
+      await cp(sourcePath, destPath, { verbatimSymlinks: true });
+    }
+  }
+}

packages/cli-v3/src/dev/devSession.ts

@@ -20,7 +20,12 @@ import { createExternalsBuildExtension, resolveAlwaysExternal } from "../build/e
 import { type DevCommandOptions } from "../commands/dev.js";
 import { eventBus } from "../utilities/eventBus.js";
 import { logger } from "../utilities/logger.js";
-import { clearTmpDirs, EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js";
+import {
+  clearTmpDirs,
+  EphemeralDirectory,
+  getStoreDir,
+  getTmpDir,
+} from "../utilities/tempDirectories.js";
 import { startDevOutput } from "./devOutput.js";
 import { startWorkerRuntime } from "./devSupervisor.js";
 import { startMcpServer, stopMcpServer } from "./mcpServer.js";
@@ -53,6 +58,8 @@ export async function startDevSession({
 }: DevSessionOptions): Promise<DevSessionInstance> {
   clearTmpDirs(rawConfig.workingDir);
   const destination = getTmpDir(rawConfig.workingDir, "build", keepTmpFiles);
+  // Create shared store directory for deduplicating chunk files across rebuilds
+  const storeDir = getStoreDir(rawConfig.workingDir);
 
   const runtime = await startWorkerRuntime({
     name,
@@ -102,6 +109,7 @@ export async function startDevSession({
       workerDir: workerDir?.path,
       environment: "dev",
       target: "dev",
+      storeDir,
     });
 
     logger.debug("Created build manifest from bundle", { buildManifest });
@@ -131,7 +139,13 @@ export async function startDevSession({
   }
 
   async function updateBuild(build: esbuild.BuildResult, workerDir: EphemeralDirectory) {
-    const bundle = await getBundleResultFromBuild("dev", rawConfig.workingDir, rawConfig, build);
+    const bundle = await getBundleResultFromBuild(
+      "dev",
+      rawConfig.workingDir,
+      rawConfig,
+      build,
+      storeDir
+    );
 
     if (bundle) {
       await updateBundle({ ...bundle, stop: undefined }, workerDir);
@@ -190,6 +204,7 @@ export async function startDevSession({
         jsxFactory: rawConfig.build.jsx.factory,
         jsxFragment: rawConfig.build.jsx.fragment,
         jsxAutomatic: rawConfig.build.jsx.automatic,
+        storeDir,
       });
 
       await updateBundle(bundleResult);

packages/cli-v3/src/utilities/fileSystem.ts

@@ -16,6 +16,77 @@ export async function createFile(
   return path;
 }
 
+/**
+ * Sanitizes a hash to be safe for use as a filename.
+ * esbuild's hashes are base64-encoded and may contain `/` and `+` characters.
+ */
+export function sanitizeHashForFilename(hash: string): string {
+  return hash.replace(/\//g, "_").replace(/\+/g, "-");
+}
+
+/**
+ * Creates a file using a content-addressable store for deduplication.
+ * Files are stored by their content hash, so identical content is only stored once.
+ * The build directory gets a hardlink to the stored file.
+ *
+ * @param filePath - The destination path for the file
+ * @param contents - The file contents to write
+ * @param storeDir - The shared store directory for deduplication
+ * @param contentHash - The content hash (e.g., from esbuild's outputFile.hash)
+ * @returns The destination file path
+ */
+export async function createFileWithStore(
+  filePath: string,
+  contents: string | NodeJS.ArrayBufferView,
+  storeDir: string,
+  contentHash: string
+): Promise<string> {
+  // Sanitize hash to be filesystem-safe (base64 can contain / and +)
+  const safeHash = sanitizeHashForFilename(contentHash);
+  // Store files by their content hash for true content-addressable storage
+  const storePath = pathModule.join(storeDir, safeHash);
+
+  // Ensure build directory exists
+  await fsModule.mkdir(pathModule.dirname(filePath), { recursive: true });
+
+  // Remove existing file at destination if it exists (hardlinks fail on existing files)
+  if (fsSync.existsSync(filePath)) {
+    await fsModule.unlink(filePath);
+  }
+
+  // Check if content already exists in store by hash
+  if (fsSync.existsSync(storePath)) {
+    // Create hardlink from build path to store path
+    // Fall back to copy if hardlink fails (e.g., on Windows or cross-device)
+    try {
+      await fsModule.link(storePath, filePath);
+    } catch (linkError) {
+      try {
+        await fsModule.copyFile(storePath, filePath);
+      } catch (copyError) {
+        throw linkError; // Rethrow original error if copy also fails
+      }
+    }
+    return filePath;
+  }
+
+  // Write to store first (using hash as filename)
+  await fsModule.writeFile(storePath, contents);
+  // Create hardlink in build directory (with original filename)
+  // Fall back to copy if hardlink fails (e.g., on Windows or cross-device)
+  try {
+    await fsModule.link(storePath, filePath);
+  } catch (linkError) {
+    try {
+      await fsModule.copyFile(storePath, filePath);
+    } catch (copyError) {
+      throw linkError; // Rethrow original error if copy also fails
+    }
+  }
+
+  return filePath;
+}
+
 export function isDirectory(configPath: string) {
   try {
     return fs.statSync(configPath).isDirectory();

packages/cli-v3/src/utilities/tempDirectories.ts

@@ -58,3 +58,15 @@ export function clearTmpDirs(projectRoot: string | undefined) {
     // This sometimes fails on Windows with EBUSY
   }
 }
+
+/**
+ * Gets the shared store directory for content-addressable build outputs.
+ * This directory persists across rebuilds and is used to deduplicate
+ * identical chunk files between build versions.
+ */
+export function getStoreDir(projectRoot: string | undefined): string {
+  projectRoot ??= process.cwd();
+  const storeDir = path.join(projectRoot, ".trigger", "tmp", "store");
+  fs.mkdirSync(storeDir, { recursive: true });
+  return storeDir;
+}

packages/core/src/v3/schemas/build.ts

@@ -68,6 +68,8 @@ export const BuildManifest = z.object({
       exclude: z.array(z.string()).optional(),
     })
     .optional(),
+  /** Maps output file paths to their content hashes for deduplication during dev */
+  outputHashes: z.record(z.string()).optional(),
 });
 
 export type BuildManifest = z.infer<typeof BuildManifest>;

references/d3-chat/src/trigger/chat.ts

@@ -199,8 +199,6 @@ export const todoChat = schemaTask({
   run: async ({ input, userId }, { signal }) => {
     metadata.set("user_id", userId);
 
-    logger.info("todoChat: starting", { input, userId });
-
     const system = `
       You are a SQL (postgres) expert who can turn natural language descriptions for a todo app 
       into a SQL query which can then be executed against a SQL database. Here is the schema: