From bc931fb93aa2e1cb3eb6169a1b939930bc743769 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Sun, 7 Jul 2024 16:59:24 +0100
Subject: [PATCH] Retry failed attempt submission with stringified error

---
 .../app/v3/services/completeAttempt.server.ts | 44 ++++++++++++++-----
 1 file changed, 34 insertions(+), 10 deletions(-)

diff --git a/apps/webapp/app/v3/services/completeAttempt.server.ts b/apps/webapp/app/v3/services/completeAttempt.server.ts
index 86e639862..698850c8b 100644
--- a/apps/webapp/app/v3/services/completeAttempt.server.ts
+++ b/apps/webapp/app/v3/services/completeAttempt.server.ts
@@ -153,15 +153,39 @@ export class CompleteAttemptService extends BaseService {
       return "COMPLETED";
     }
 
-    await this._prisma.taskRunAttempt.update({
-      where: { id: taskRunAttempt.id },
-      data: {
-        status: "FAILED",
-        completedAt: new Date(),
-        error: completion.error,
-        usageDurationMs: completion.usage?.durationMs,
-      },
-    });
+    let attemptError = completion.error;
+
+    try {
+      await this._prisma.taskRunAttempt.update({
+        where: { id: taskRunAttempt.id },
+        data: {
+          status: "FAILED",
+          completedAt: new Date(),
+          error: attemptError,
+          usageDurationMs: completion.usage?.durationMs,
+        },
+      });
+    } catch (error) {
+      logger.error("Failed to update task run attempt after failure, trying again", {
+        error,
+        taskRunAttempt,
+      });
+
+      attemptError = {
+        type: "STRING_ERROR",
+        raw: JSON.stringify(completion.error),
+      };
+
+      await this._prisma.taskRunAttempt.update({
+        where: { id: taskRunAttempt.id },
+        data: {
+          status: "FAILED",
+          completedAt: new Date(),
+          error: attemptError,
+          usageDurationMs: completion.usage?.durationMs,
+        },
+      });
+    }
 
     const environment = env ?? (await this.#getEnvironment(execution.environment.id));
 
@@ -276,7 +300,7 @@ export class CompleteAttemptService extends BaseService {
             name: "exception",
             time: new Date(),
             properties: {
-              exception: createExceptionPropertiesFromError(completion.error),
+              exception: createExceptionPropertiesFromError(attemptError),
             },
           },
         ],