fix(deployments): retry transient depot build init failures (#2586)

The Depot build init with `depot.build.v1.BuildService.createBuild` fails surprisingly often due to transient errors, causing the whole deployment to fail. This PR adds a simple retry mechanism with backoff using p-retry. This should improve the failure rate.

apps/webapp/app/routes/api.v1.deployments.$deploymentId.progress.ts

@@ -55,8 +55,10 @@ export async function action({ request, params }: ActionFunctionArgs) {
       },
       (error) => {
         switch (error.type) {
-          case "failed_to_extend_deployment_timeout":
+          case "failed_to_extend_deployment_timeout": {
+            logger.warn("Failed to extend deployment timeout", { error: error.cause });
             return new Response(null, { status: 204 }); // ignore these errors for now
+          }
           case "deployment_not_found":
             return json({ error: "Deployment not found" }, { status: 404 });
           case "deployment_cannot_be_progressed":
@@ -64,8 +66,10 @@ export async function action({ request, params }: ActionFunctionArgs) {
               { error: "Deployment is not in a progressable state (PENDING or INSTALLING)" },
               { status: 409 }
             );
-          case "failed_to_create_remote_build":
+          case "failed_to_create_remote_build": {
+            logger.error("Failed to create remote Depot build", { error: error.cause });
             return json({ error: "Failed to create remote build" }, { status: 500 });
+          }
           case "other":
           default:
             error.type satisfies "other";

apps/webapp/app/v3/remoteImageBuilder.server.ts

@@ -3,6 +3,8 @@ import { type ExternalBuildData } from "@trigger.dev/core/v3";
 import { type Project } from "@trigger.dev/database";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
+import pRetry from "p-retry";
+import { logger } from "~/services/logger.server";
 
 export async function createRemoteImageBuild(
   project: Project
@@ -13,11 +15,22 @@ export async function createRemoteImageBuild(
 
   const builderProjectId = await createBuilderProjectIfNotExists(project);
 
-  const result = await depot.build.v1.BuildService.createBuild(
-    { projectId: builderProjectId },
+  const result = await pRetry(
+    () =>
+      depot.build.v1.BuildService.createBuild(
+        { projectId: builderProjectId },
+        {
+          headers: {
+            Authorization: `Bearer ${env.DEPOT_TOKEN}`,
+          },
+        }
+      ),
     {
-      headers: {
-        Authorization: `Bearer ${env.DEPOT_TOKEN}`,
+      retries: 3,
+      minTimeout: 200,
+      maxTimeout: 2000,
+      onFailedAttempt: (error) => {
+        logger.error("Failed attempt to create remote Depot build", { error });
       },
     }
   );

apps/webapp/package.json

@@ -166,6 +166,7 @@
     "openai": "^4.33.1",
     "p-limit": "^6.2.0",
     "p-map": "^6.0.0",
+    "p-retry": "^4.6.1",
     "parse-duration": "^2.1.0",
     "posthog-js": "^1.93.3",
     "posthog-node": "4.17.1",

pnpm-lock.yaml

@@ -605,6 +605,9 @@ importers:
       p-map:
         specifier: ^6.0.0
         version: 6.0.0
+      p-retry:
+        specifier: ^4.6.1
+        version: 4.6.2
       parse-duration:
         specifier: ^2.1.0
         version: 2.1.4