github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

getting closer to executing dev runs...

Browse Source
This commit is contained in:
Eric Allam
2024-08-08 15:40:33 +01:00
committed by Eric Allam
parent de81f046b2
commit ff9013bf01
10 changed files with 202 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli-v3/src/build/bundle.ts
+26 -9
View File
@@ -14,6 +14,8 @@ import {
shims,
} from "./packageModules.js";
import { buildPlugins } from "./plugins.js";
import { createHash, hash } from "node:crypto";
import { createFile } from "../utilities/fileSystem.js";
export interface BundleOptions {
target: BuildTarget;
@@ -28,6 +30,7 @@ export interface BundleOptions {
}
export type BundleResult = {
contentHash: string;
files: TaskFile[];
configPath: string;
loaderEntryPoint: string | undefined;
@@ -62,7 +65,7 @@ export async function bundleWorker(options: BundleOptions): Promise<BundleResult
absWorkingDir: options.cwd,
bundle: true,
metafile: true,
write: true,
write: false,
minify: false,
splitting: true,
charset: "utf8",
@@ -114,7 +117,7 @@ export async function bundleWorker(options: BundleOptions): Promise<BundleResult
stop = async function () {};
}
const bundleResult = getBundleResultFromBuild(options.target, options.cwd, result);
const bundleResult = await getBundleResultFromBuild(options.target, options.cwd, result);
if (!bundleResult) {
throw new Error("Failed to get bundle result");
@@ -123,11 +126,19 @@ export async function bundleWorker(options: BundleOptions): Promise<BundleResult
return { ...bundleResult, stop };
}
export function getBundleResultFromBuild(
export async function getBundleResultFromBuild(
target: BuildTarget,
workingDir: string,
result: esbuild.BuildResult<{ metafile: true }>
): Omit<BundleResult, "stop"> | undefined {
result: esbuild.BuildResult<{ metafile: true; write: false }>
): Promise<Omit<BundleResult, "stop"> | undefined> {
const hasher = createHash("md5");
for (const outputFile of result.outputFiles) {
hasher.update(outputFile.hash);
await createFile(outputFile.path, outputFile.contents);
}
const files: Array<{ entry: string; out: string }> = [];
let configPath: string | undefined;
@@ -149,10 +160,15 @@ export function getBundleResultFromBuild(
} else if (isEntryPointForTarget(outputMeta.entryPoint, target)) {
workerEntryPoint = $outputPath;
} else {
files.push({
entry: outputMeta.entryPoint,
out: $outputPath,
});
if (
!outputMeta.entryPoint.startsWith("..") &&
!outputMeta.entryPoint.includes("node_modules")
) {
files.push({
entry: outputMeta.entryPoint,
out: $outputPath,
});
}
}
}
}
@@ -166,6 +182,7 @@ export function getBundleResultFromBuild(
configPath: configPath,
loaderEntryPoint,
workerEntryPoint,
contentHash: hasher.digest("hex"),
};
}
packages/cli-v3/src/build/externals.ts
+62 -84
View File
@@ -1,4 +1,3 @@
import * as esbuild from "esbuild";
import { makeRe } from "minimatch";
import { mkdir, symlink } from "node:fs/promises";
@@ -20,10 +19,7 @@ const FORCED_EXTERNALS = ["import-in-the-middle"];
* This function will create a symbolic link from a place where the external is resolvable
* to the actual resolved external path
*/
async function linkUnresolvableExternals(
externals: Array<CollectedExternal>,
resolveDir: string
) {
async function linkUnresolvableExternals(externals: Array<CollectedExternal>, resolveDir: string) {
for (const external of externals) {
if (!(await isExternalResolvable(external, resolveDir))) {
logger.debug("External is not resolvable", { external });
@@ -44,10 +40,7 @@ async function linkExternal(external: CollectedExternal, resolveDir: string) {
await symlink(external.path, join(destinationPath, external.name), "dir");
}
async function isExternalResolvable(
external: CollectedExternal,
resolveDir: string
) {
async function isExternalResolvable(external: CollectedExternal, resolveDir: string) {
try {
const resolvedPath = nodeResolve.sync(external.name, {
basedir: resolveDir,
@@ -60,7 +53,6 @@ async function isExternalResolvable(
return true;
} catch (e) {
console.error("Could not resolve external", { external, resolveDir, e });
return false;
}
}
@@ -100,70 +92,65 @@ function createExternalsCollector(
});
maybeExternals.forEach((external) => {
build.onResolve(
{ filter: external.filter, namespace: "file" },
async (args) => {
const resolvedPath = nodeResolve.sync(args.path, {
basedir: args.resolveDir,
});
build.onResolve({ filter: external.filter, namespace: "file" }, async (args) => {
const resolvedPath = nodeResolve.sync(args.path, {
basedir: args.resolveDir,
});
logger.debug("Resolved external", {
external,
resolvedPath,
args,
});
logger.debug("Resolved external", {
external,
resolvedPath,
args,
});
const packageJsonPath = await resolvePackageJSON(
dirname(resolvedPath)
);
const packageJsonPath = await resolvePackageJSON(dirname(resolvedPath));
if (!packageJsonPath) {
return undefined;
}
logger.debug("Found package.json", { packageJsonPath });
const packageJson = await readPackageJSON(packageJsonPath);
if (!packageJson || !packageJson.name) {
return undefined;
}
if (!external.filter.test(packageJson.name)) {
logger.debug("Package name does not match", {
external,
packageJson,
});
return undefined;
}
if (!packageJson.version) {
logger.debug("No version found in package.json", {
external,
packageJson,
});
return undefined;
}
externals.push({
name: packageJson.name,
path: dirname(packageJsonPath),
version: packageJson.version,
});
logger.debug("Resolved external", {
external,
resolvedPath,
args,
});
return {
external: true,
};
if (!packageJsonPath) {
return undefined;
}
);
logger.debug("Found package.json", { packageJsonPath });
const packageJson = await readPackageJSON(packageJsonPath);
if (!packageJson || !packageJson.name) {
return undefined;
}
if (!external.filter.test(packageJson.name)) {
logger.debug("Package name does not match", {
external,
packageJson,
});
return undefined;
}
if (!packageJson.version) {
logger.debug("No version found in package.json", {
external,
packageJson,
});
return undefined;
}
externals.push({
name: packageJson.name,
path: dirname(packageJsonPath),
version: packageJson.version,
});
logger.debug("Resolved external", {
external,
resolvedPath,
args,
});
return {
external: true,
};
});
});
},
},
@@ -172,10 +159,7 @@ function createExternalsCollector(
type MaybeExternal = { raw: string; filter: RegExp };
function discoverMaybeExternals(
target: BuildTarget,
config: ResolvedConfig
): Array<MaybeExternal> {
function discoverMaybeExternals(target: BuildTarget, config: ResolvedConfig): Array<MaybeExternal> {
const external: Array<MaybeExternal> = [];
for (const externalName of FORCED_EXTERNALS) {
@@ -187,9 +171,7 @@ function discoverMaybeExternals(
external.push({
raw: externalName,
filter: new RegExp(
`^${externalName}$|${externalRegex.source}`
),
filter: new RegExp(`^${externalName}$|${externalRegex.source}`),
});
}
@@ -212,14 +194,12 @@ function discoverMaybeExternals(
const externalRegex = makeRe(externalName);
if (!externalRegex) {
continue
continue;
}
external.push({
raw: externalName,
filter: new RegExp(
`^${externalName}$|${externalRegex.source}`
),
filter: new RegExp(`^${externalName}$|${externalRegex.source}`),
});
}
@@ -235,9 +215,7 @@ function discoverMaybeExternals(
external.push({
raw: externalName,
filter: new RegExp(
`^${externalName}$|${externalRegex.source}`
),
filter: new RegExp(`^${externalName}$|${externalRegex.source}`),
});
}
}
packages/cli-v3/src/dev/backgroundWorker.ts
+17 -5
View File
@@ -115,9 +115,13 @@ export class BackgroundWorkerCoordinator {
await worker.cancelRun(taskRunId);
}
async registerWorker(record: CreateBackgroundWorkerResponse, worker: BackgroundWorker) {
async registerWorker(worker: BackgroundWorker) {
if (!worker.serverWorker) {
return;
}
for (const [workerId, existingWorker] of this._backgroundWorkers.entries()) {
if (workerId === record.id) {
if (workerId === worker.serverWorker.id) {
continue;
}
@@ -125,11 +129,19 @@ export class BackgroundWorkerCoordinator {
this.onWorkerDeprecated.post({ worker: existingWorker, id: workerId });
}
this._backgroundWorkers.set(record.id, worker);
this.onWorkerRegistered.post({ worker, id: record.id, record });
this._backgroundWorkers.set(worker.serverWorker.id, worker);
this.onWorkerRegistered.post({
worker,
id: worker.serverWorker.id,
record: worker.serverWorker,
});
worker.onTaskRunHeartbeat.attach((id) => {
this.onWorkerTaskRunHeartbeat.post({ id, backgroundWorkerId: record.id, worker });
this.onWorkerTaskRunHeartbeat.post({
id,
backgroundWorkerId: worker.serverWorker!.id,
worker,
});
});
}
packages/cli-v3/src/dev/devSession.ts
+10 -2
View File
@@ -22,6 +22,7 @@ import { logger } from "../utilities/logger.js";
import { EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js";
import { copyManifestToDir } from "../build/manifests.js";
import { startWorkerRuntime } from "./workerRuntime.js";
import { chalkGrey } from "../utilities/cliOutput.js";
export type DevSessionOptions = {
name: string | undefined;
@@ -78,7 +79,7 @@ export async function startDevSession({
}
async function updateBuild(build: esbuild.BuildResult, workerDir: EphemeralDirectory) {
const bundle = getBundleResultFromBuild("dev", rawConfig.workingDir, build);
const bundle = await getBundleResultFromBuild("dev", rawConfig.workingDir, build);
if (bundle) {
await updateBundle({ ...bundle, stop: undefined }, workerDir);
@@ -91,6 +92,10 @@ export async function startDevSession({
setup(b: esbuild.PluginBuild) {
b.onStart(() => {
logger.debug("on-end plugin started");
if (bundled) {
logger.log(chalkGrey("○ Rebuilding background worker…"));
}
});
b.onEnd(async (result: esbuild.BuildResult) => {
const errors = result.errors;
@@ -108,7 +113,7 @@ export async function startDevSession({
// First bundle, no need to update bundle
bundled = true;
} else {
const workerDir = getTmpDir(rawConfig.workingDir, "worker");
const workerDir = getTmpDir(rawConfig.workingDir, "build");
await updateBuild(result, workerDir);
}
@@ -129,6 +134,8 @@ export async function startDevSession({
jsxAutomatic: rawConfig.build.jsx.automatic,
});
logger.log(chalkGrey("○ Building background worker…"));
await updateBundle(bundleResult);
return bundleResult.stop;
@@ -154,6 +161,7 @@ async function createBuildManifestFromBundle(
workerDir: string | undefined
): Promise<BuildManifest> {
const buildManifest: BuildManifest = {
contentHash: bundle.contentHash,
runtime: resolvedConfig.runtime ?? DEFAULT_RUNTIME,
target: "dev",
files: bundle.files,
packages/cli-v3/src/dev/workerRuntime.ts
+81 -1
View File
@@ -1,9 +1,12 @@
import {
BuildManifest,
clientWebsocketMessages,
CreateBackgroundWorkerRequestBody,
SemanticInternalAttributes,
serverWebsocketMessages,
TaskManifest,
TaskRunExecutionLazyAttemptPayload,
WorkerManifest,
} from "@trigger.dev/core/v3";
import { ResolvedConfig } from "@trigger.dev/core/v3/build";
import { ClientRequestArgs } from "node:http";
@@ -11,7 +14,7 @@ import { WebSocket } from "partysocket";
import { ClientOptions, WebSocket as wsWebSocket } from "ws";
import { CliApiClient } from "../apiClient.js";
import { DevCommandOptions } from "../commands/dev.js";
import { chalkError } from "../utilities/cliOutput.js";
import { chalkError, chalkGrey, chalkTask } from "../utilities/cliOutput.js";
import { logger } from "../utilities/logger.js";
import { BackgroundWorker, BackgroundWorkerCoordinator } from "./backgroundWorker.js";
import {
@@ -20,6 +23,7 @@ import {
ZodMessageSender,
} from "@trigger.dev/core/v3/zodMessageHandler";
import { resolveDotEnvVars } from "../utilities/dotEnv.js";
import { VERSION } from "../version.js";
export interface WorkerRuntime {
shutdown(): Promise<void>;
@@ -47,6 +51,7 @@ class DevWorkerRuntime implements WorkerRuntime {
private backgroundWorkerCoordinator: BackgroundWorkerCoordinator;
private sender: ZodMessageSender<typeof clientWebsocketMessages>;
private websocketMessageHandler: ZodMessageHandler<typeof serverWebsocketMessages>;
private lastBuild: BuildManifest | undefined;
constructor(public readonly options: WorkerRuntimeOptions) {
const websocketUrl = new URL(this.options.client.apiURL);
@@ -151,6 +156,11 @@ class DevWorkerRuntime implements WorkerRuntime {
}
async initializeWorker(manifest: BuildManifest, options?: { cwd?: string }): Promise<void> {
if (this.lastBuild && this.lastBuild.contentHash === manifest.contentHash) {
logger.log(chalkGrey("○ No changes detected, skipping build…"));
return;
}
const env = await this.#getEnvVars();
const backgroundWorker = new BackgroundWorker(manifest, {
@@ -159,6 +169,41 @@ class DevWorkerRuntime implements WorkerRuntime {
});
await backgroundWorker.initialize();
if (!backgroundWorker.manifest) {
throw new Error("Could not initialize worker");
}
const issues = validateWorkerManifest(backgroundWorker.manifest);
if (issues.length > 0) {
issues.forEach((issue) => logger.error(issue));
return;
}
const backgroundWorkerBody: CreateBackgroundWorkerRequestBody = {
localOnly: true,
metadata: {
packageVersion: VERSION,
cliPackageVersion: VERSION,
tasks: backgroundWorker.manifest.tasks,
contentHash: manifest.contentHash,
},
supportsLazyAttempts: true,
};
const backgroundWorkerRecord = await this.options.client.createBackgroundWorker(
this.options.config.project,
backgroundWorkerBody
);
if (!backgroundWorkerRecord.success) {
throw new Error(backgroundWorkerRecord.error);
}
backgroundWorker.serverWorker = backgroundWorkerRecord.data;
this.backgroundWorkerCoordinator.registerWorker(backgroundWorker);
this.lastBuild = manifest;
}
async #getEnvVars(): Promise<Record<string, string>> {
@@ -270,3 +315,38 @@ function gatherProcessEnv() {
// Filter out undefined values
return Object.fromEntries(Object.entries(env).filter(([key, value]) => value !== undefined));
}
function validateWorkerManifest(manifest: WorkerManifest): string[] {
const issues: string[] = [];
if (!manifest.tasks || manifest.tasks.length === 0) {
issues.push("No tasks defined. Make sure you are exporting tasks.");
}
// Check for any duplicate task ids
const taskIds = manifest.tasks.map((task) => task.id);
const duplicateTaskIds = taskIds.filter((id, index) => taskIds.indexOf(id) !== index);
if (duplicateTaskIds.length > 0) {
issues.push(createDuplicateTaskIdOutputErrorMessage(duplicateTaskIds, manifest.tasks));
}
return issues;
}
function createDuplicateTaskIdOutputErrorMessage(
duplicateTaskIds: Array<string>,
tasks: Array<TaskManifest>
) {
const duplicateTable = duplicateTaskIds
.map((id) => {
const $tasks = tasks.filter((task) => task.id === id);
return `\n\n${chalkTask(id)} was found in:${tasks
.map((task) => `\n${task.filePath} -> ${task.exportName}`)
.join("")}`;
})
.join("");
return `Duplicate ${chalkTask("task id")} detected:${duplicateTable}`;
}
packages/cli-v3/src/entryPoints/dev.ts
+1 -2
View File
@@ -120,8 +120,7 @@ async function registerTaskFileMetadata(files: Array<{ entry: string; out: strin
if (taskCatalog.taskExists(task.id)) {
taskCatalog.registerTaskFileMetadata(task.id, {
exportName,
filePath: file.out,
entryPoint: file.entry,
filePath: file.entry,
});
}
}
packages/cli-v3/src/utilities/fileSystem.ts
+4 -1
View File
@@ -5,7 +5,10 @@ import { tmpdir } from "node:os";
import pathModule from "node:path";
// Creates a file at the given path, if the directory doesn't exist it will be created
export async function createFile(path: string, contents: string): Promise<string> {
export async function createFile(
path: string,
contents: string | NodeJS.ArrayBufferView
): Promise<string> {
await fsModule.mkdir(pathModule.dirname(path), { recursive: true });
await fsModule.writeFile(path, contents);
packages/core/src/v3/extensions/emitDecoratorMetadata.ts
-4
View File
@@ -34,10 +34,6 @@ function plugin(options: EmitDecoratorMetadataOptions = {}): esbuild.Plugin {
return;
}
console.log("Setting up typescript decorators plugin", {
tsconfig,
});
build.onLoad({ filter: /\.ts$/ }, async (args) => {
const ts = await readFile(args.path, "utf8");
packages/core/src/v3/schemas/build.ts
+1
View File
@@ -19,6 +19,7 @@ export type BuildRuntime = z.infer<typeof BuildRuntime>;
export const BuildManifest = z.object({
target: BuildTarget,
contentHash: z.string(),
runtime: BuildRuntime,
config: ConfigManifest,
files: z.array(TaskFile),
packages/core/src/v3/schemas/schemas.ts
-1
View File
@@ -169,7 +169,6 @@ export type TaskFile = z.infer<typeof TaskFile>;
const taskFileMetadata = {
filePath: z.string(),
entryPoint: z.string(),
exportName: z.string(),
};