v3: various small fixes (#1192)

* Await file watcher cleanup in dev

* Fix artifact detection logs

* Fix next runs table when schedule disabled

* Improve OOM error messages

* Add test link to completed deployment message

* Fix OOM detection, again

* Add changeset

.changeset/purple-spiders-care.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Await file watcher cleanup in dev

.changeset/tall-masks-repeat.md

@@ -0,0 +1,9 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Fix artifact detection logs
+- Fix OOM detection and error messages
+- Add test link to cli deployment completion

apps/kubernetes-provider/src/index.ts

@@ -529,27 +529,27 @@ provider.listen();
 
 const taskMonitor = new TaskMonitor({
   runtimeEnv: RUNTIME_ENV,
-  onIndexFailure: async (deploymentId, failureInfo) => {
-    logger.log("Indexing failed", { deploymentId, failureInfo });
+  onIndexFailure: async (deploymentId, details) => {
+    logger.log("Indexing failed", { deploymentId, details });
 
     try {
       provider.platformSocket.send("INDEXING_FAILED", {
         deploymentId,
         error: {
-          name: `Crashed with exit code ${failureInfo.exitCode}`,
-          message: failureInfo.reason,
-          stack: failureInfo.logs,
+          name: `Crashed with exit code ${details.exitCode}`,
+          message: details.reason,
+          stack: details.logs,
         },
       });
     } catch (error) {
       logger.error(error);
     }
   },
-  onRunFailure: async (runId, failureInfo) => {
-    logger.log("Run failed:", { runId, failureInfo });
+  onRunFailure: async (runId, details) => {
+    logger.log("Run failed:", { runId, details });
 
     try {
-      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
     } catch (error) {
       logger.error(error);
     }

apps/kubernetes-provider/src/taskMonitor.ts

@@ -1,25 +1,20 @@
 import * as k8s from "@kubernetes/client-node";
 import { SimpleLogger } from "@trigger.dev/core-apps";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core-apps/process";
 import { setTimeout } from "timers/promises";
 import PQueue from "p-queue";
+import type { Prettify } from "@trigger.dev/core/v3";
 
-type IndexFailureHandler = (
-  deploymentId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type FailureDetails = Prettify<{
+  exitCode: number;
+  reason: string;
+  logs: string;
+  overrideCompletion: boolean;
+}>;
 
-type RunFailureHandler = (
-  runId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
+
+type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
 
 type TaskMonitorOptions = {
   runtimeEnv: "local" | "kubernetes";
@@ -144,8 +139,7 @@ export class TaskMonitor {
     const containerState = this.#getContainerStateSummary(containerStatus.state);
     const exitCode = containerState.exitCode ?? -1;
 
-    // We use this special exit code to signal any errors were already handled elsewhere
-    if (exitCode === 111) {
+    if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
       return;
     }
 
@@ -162,6 +156,7 @@ export class TaskMonitor {
 
     let reason = rawReason || "Unknown error";
     let logs = rawLogs || "";
+    let overrideCompletion = false;
 
     switch (rawReason) {
       case "Error":
@@ -181,8 +176,10 @@ export class TaskMonitor {
         }
         break;
       case "OOMKilled":
-        reason =
-          "Process ran out of memory! Try choosing a machine preset with more memory for this task.";
+        overrideCompletion = true;
+        reason = `${
+          exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
+        } ran out of memory! Try choosing a machine preset with more memory for this task.`;
         break;
       default:
         break;
@@ -192,7 +189,8 @@ export class TaskMonitor {
       exitCode,
       reason,
       logs,
-    };
+      overrideCompletion,
+    } satisfies FailureDetails;
 
     const app = pod.metadata?.labels?.app;
 

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -125,7 +125,7 @@ export function TaskRunsTable({
       </TableHeader>
       <TableBody>
         {total === 0 && !hasFilters ? (
-          <TableBlankRow colSpan={9}>
+          <TableBlankRow colSpan={10}>
             {!isLoading && <NoRuns title="No runs found" />}
           </TableBlankRow>
         ) : runs.length === 0 ? (

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules.$scheduleParam/route.tsx

@@ -23,6 +23,7 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
   Table,
+  TableBlankRow,
   TableBody,
   TableCell,
   TableHeader,
@@ -180,6 +181,14 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   }
 };
 
+function PlaceholderText({ title }: { title: string }) {
+  return (
+    <div className="flex items-center justify-center">
+      <Paragraph className="w-auto">{title}</Paragraph>
+    </div>
+  );
+}
+
 export default function Page() {
   const { schedule } = useTypedLoaderData<typeof loader>();
   const location = useLocation();
@@ -252,18 +261,30 @@ export default function Page() {
                   </TableRow>
                 </TableHeader>
                 <TableBody>
-                  {schedule.nextRuns.map((run, index) => (
-                    <TableRow key={index}>
-                      {!isUtc && (
-                        <TableCell>
-                          <DateTime date={run} timeZone={schedule.timezone} />
-                        </TableCell>
-                      )}
-                      <TableCell>
-                        <DateTime date={run} timeZone="UTC" />
-                      </TableCell>
-                    </TableRow>
-                  ))}
+                  {schedule.active ? (
+                    schedule.nextRuns.length ? (
+                      schedule.nextRuns.map((run, index) => (
+                        <TableRow key={index}>
+                          {!isUtc && (
+                            <TableCell>
+                              <DateTime date={run} timeZone={schedule.timezone} />
+                            </TableCell>
+                          )}
+                          <TableCell>
+                            <DateTime date={run} timeZone="UTC" />
+                          </TableCell>
+                        </TableRow>
+                      ))
+                    ) : (
+                      <TableBlankRow colSpan={1}>
+                        <PlaceholderText title="You found a bug" />
+                      </TableBlankRow>
+                    )
+                  ) : (
+                    <TableBlankRow colSpan={1}>
+                      <PlaceholderText title="Schedule disabled" />
+                    </TableBlankRow>
+                  )}
                 </TableBody>
               </Table>
             </div>

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -13,6 +13,7 @@ export type CrashTaskRunServiceOptions = {
   logs?: string;
   crashAttempts?: boolean;
   crashedAt?: Date;
+  overrideCompletion?: boolean;
 };
 
 export class CrashTaskRunService extends BaseService {
@@ -36,7 +37,7 @@ export class CrashTaskRunService extends BaseService {
     }
 
     // Make sure the task run is in a crashable state
-    if (!isCrashableRunStatus(taskRun.status)) {
+    if (!opts.overrideCompletion && !isCrashableRunStatus(taskRun.status)) {
       logger.error("Task run is not in a crashable state", { runId, status: taskRun.status });
       return;
     }

apps/webapp/app/v3/services/deploymentIndexFailed.server.ts

@@ -18,7 +18,8 @@ export class DeploymentIndexFailed extends BaseService {
       message: string;
       stack?: string;
       stderr?: string;
-    }
+    },
+    overrideCompletion = false
   ) {
     const isFriendlyId = maybeFriendlyId.startsWith("deployment_");
 
@@ -38,6 +39,15 @@ export class DeploymentIndexFailed extends BaseService {
     }
 
     if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
+      if (overrideCompletion) {
+        logger.error("No support for overriding final deployment statuses just yet", {
+          id: deployment.id,
+          status: deployment.status,
+          previousError: deployment.errorData,
+          incomingError: error,
+        });
+      }
+
       logger.error("Worker deployment already in final state", {
         id: deployment.id,
         status: deployment.status,

packages/cli-v3/src/commands/deploy.ts

@@ -441,6 +441,13 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
     `${authorization.dashboardUrl}/projects/v3/${resolvedConfig.config.project}/deployments/${finishedDeployment.shortCode}`
   );
 
+  const testLink = cliLink(
+    "Test tasks",
+    `${authorization.dashboardUrl}/projects/v3/${resolvedConfig.config.project}/test?environment=${
+      options.env === "prod" ? "prod" : "stg"
+    }`
+  );
+
   switch (finishedDeployment.status) {
     case "DEPLOYED": {
       if (warnings.warnings.length > 0) {
@@ -461,7 +468,7 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
         outro(
           `Version ${version} deployed with ${taskCount} detected task${
             taskCount === 1 ? "" : "s"
-          } ${deploymentLink}`
+          } | ${deploymentLink} | ${testLink}`
         );
       }
 

packages/cli-v3/src/commands/dev.tsx

@@ -760,15 +760,23 @@ function useDev({
     });
 
     return () => {
-      logger.debug(`Shutting down dev session for ${config.project}`);
+      const cleanup = async () => {
+        logger.debug(`Shutting down dev session for ${config.project}`);
 
-      taskFileWatcher.close();
+        const start = Date.now();
 
-      websocket?.close();
-      backgroundWorkerCoordinator.close();
-      ctx?.dispose().catch((error) => {
-        console.error(error);
-      });
+        await taskFileWatcher.close();
+
+        websocket?.close();
+        backgroundWorkerCoordinator.close();
+        ctx?.dispose().catch((error) => {
+          console.error(error);
+        });
+
+        logger.debug(`Shutdown completed in ${Date.now() - start}ms`);
+      };
+
+      cleanup();
     };
   }, [config, apiUrl, apiKey, environmentClient]);
 }

packages/cli-v3/src/utilities/getUserPackageManager.ts

@@ -62,6 +62,7 @@ export async function detectPackageManagerFromArtifacts(path: string): Promise<P
     case LOCKFILES.npm:
     case LOCKFILES.npmShrinkwrap:
       logger.debug("Found npm artifact", { foundPath });
+      return "npm";
     case LOCKFILES.bun:
       logger.debug("Found bun artifact", { foundPath });
       return "npm";

packages/cli-v3/src/workers/common/errors.ts

@@ -69,7 +69,8 @@ export class GracefulExitTimeoutError extends Error {
 export function getFriendlyErrorMessage(
   code: number,
   signal: NodeJS.Signals | null,
-  stderr: string | undefined
+  stderr: string | undefined,
+  dockerMode = true
 ) {
   const message = (text: string) => {
     if (signal) {
@@ -79,7 +80,20 @@ export function getFriendlyErrorMessage(
     }
   };
 
-  if (code === 137 || stderr?.includes("OOMErrorHandler")) {
+  if (code === 137) {
+    if (dockerMode) {
+      return message(
+        "Process ran out of memory! Try choosing a machine preset with more memory for this task."
+      );
+    } else {
+      // Note: containerState reason and message should be checked to clarify the error
+      return message(
+        "Process most likely ran out of memory, but we can't be certain. Try choosing a machine preset with more memory for this task."
+      );
+    }
+  }
+
+  if (stderr?.includes("OOMErrorHandler")) {
     return message(
       "Process ran out of memory! Try choosing a machine preset with more memory for this task."
     );

packages/cli-v3/src/workers/prod/entry-point.ts

@@ -5,12 +5,14 @@ import {
   PreStopCauses,
   ProdWorkerToCoordinatorMessages,
   TaskResource,
+  TaskRunErrorCodes,
   TaskRunFailedExecutionResult,
   WaitReason,
 } from "@trigger.dev/core/v3";
 import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
 import { HttpReply, getRandomPortNumber } from "@trigger.dev/core-apps/http";
 import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core-apps/process";
 import { readFile } from "node:fs/promises";
 import { createServer } from "node:http";
 import { ProdBackgroundWorker } from "./backgroundWorker";
@@ -97,12 +99,12 @@ class ProdWorker {
     logger.log("Unhandled signal", { signal });
   }
 
-  async #exitGracefully(gracefulExitTimeoutElapsed = false) {
+  async #exitGracefully(gracefulExitTimeoutElapsed = false, exitCode = 0) {
     await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
 
     if (!gracefulExitTimeoutElapsed) {
       // TODO: Maybe add a sensible timeout instead of a conditional to avoid zombies
-      process.exit(0);
+      process.exit(exitCode);
     }
   }
 
@@ -315,7 +317,11 @@ class ProdWorker {
     }
   }
 
-  async #prepareForRetry(willCheckpointAndRestore: boolean, shouldExit: boolean) {
+  async #prepareForRetry(
+    willCheckpointAndRestore: boolean,
+    shouldExit: boolean,
+    exitCode?: number
+  ) {
     logger.log("prepare for retry", { willCheckpointAndRestore, shouldExit });
 
     // Graceful shutdown on final attempt
@@ -324,7 +330,7 @@ class ProdWorker {
         logger.log("WARNING: Will checkpoint but also requested exit. This won't end well.");
       }
 
-      await this.#exitGracefully();
+      await this.#exitGracefully(false, exitCode);
       return;
     }
 
@@ -516,7 +522,14 @@ class ProdWorker {
 
             logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
 
-            this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
+            const exitCode =
+              !completion.ok &&
+              completion.error.type === "INTERNAL_ERROR" &&
+              completion.error.code === TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE
+                ? EXIT_CODE_CHILD_NONZERO
+                : 0;
+
+            this.#prepareForRetry(willCheckpointAndRestore, shouldExit, exitCode);
           } catch (error) {
             const completion: TaskRunFailedExecutionResult = {
               ok: false,
@@ -709,8 +722,8 @@ class ProdWorker {
             }
 
             await setTimeout(200);
-            // Use exit code 111 so we can ignore those failures in the task monitor
-            process.exit(111);
+
+            process.exit(EXIT_CODE_ALREADY_HANDLED);
           }
         }
 

packages/core-apps/src/process.ts

@@ -0,0 +1,2 @@
+export const EXIT_CODE_ALREADY_HANDLED = 111;
+export const EXIT_CODE_CHILD_NONZERO = 112;

packages/core-apps/src/provider.ts

@@ -14,6 +14,7 @@ import { getRandomPortNumber, HttpReply, getTextBody } from "./http";
 import { SimpleLogger } from "./logger";
 import { isExecaChildProcess } from "./checkpoints";
 import { setTimeout } from "node:timers/promises";
+import { EXIT_CODE_ALREADY_HANDLED } from "./process";
 
 const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
 const MACHINE_NAME = process.env.MACHINE_NAME || "local";
@@ -198,7 +199,7 @@ export class ProviderShell implements Provider {
                 stderr: error.stderr,
               });
 
-              if (error.exitCode === 111) {
+              if (error.exitCode === EXIT_CODE_ALREADY_HANDLED) {
                 logger.error("Index failure already reported by the worker", {
                   socketMessage: message,
                 });

packages/core/src/v3/schemas/messages.ts

@@ -334,6 +334,7 @@ export const ProviderToPlatformMessages = {
       exitCode: z.number().optional(),
       message: z.string().optional(),
       logs: z.string().optional(),
+      overrideCompletion: z.boolean().optional(),
     }),
   },
   INDEXING_FAILED: {
@@ -346,6 +347,7 @@ export const ProviderToPlatformMessages = {
         stack: z.string().optional(),
         stderr: z.string().optional(),
       }),
+      overrideCompletion: z.string().optional(),
     }),
   },
 };