From ed7342bf49e514e06ba98081dcdd2f557fc19532 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Tue, 19 Mar 2024 13:49:42 +0000
Subject: [PATCH] rethrow k8s errors after logging

---
 apps/kubernetes-provider/src/index.ts | 54 ++++++++++++++++-----------
 packages/core-apps/src/provider.ts    | 18 +++++----
 2 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts
index 66ea4c510..851bc3ecd 100644
--- a/apps/kubernetes-provider/src/index.ts
+++ b/apps/kubernetes-provider/src/index.ts
@@ -338,12 +338,8 @@ class KubernetesTaskOperations implements TaskOperations {
     try {
       const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod);
       logger.debug(res.body);
-    } catch (err: any) {
-      if ("body" in err) {
-        logger.error(err.body);
-      } else {
-        logger.error(err);
-      }
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
     }
   }
 
@@ -354,12 +350,8 @@ class KubernetesTaskOperations implements TaskOperations {
         opts.namespace.metadata.name
       );
       logger.debug(res.body);
-    } catch (err: any) {
-      if ("body" in err) {
-        logger.error(err.body);
-      } else {
-        logger.error(err);
-      }
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
     }
   }
 
@@ -368,12 +360,8 @@ class KubernetesTaskOperations implements TaskOperations {
       const res = await this.#k8sApi.core.readNamespacedPod(podName, namespace.metadata.name);
       logger.debug(res.body);
       return res.body;
-    } catch (err: any) {
-      if ("body" in err) {
-        logger.error(err.body);
-      } else {
-        logger.error(err);
-      }
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
     }
   }
 
@@ -381,12 +369,34 @@ class KubernetesTaskOperations implements TaskOperations {
     try {
       const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job);
       logger.debug(res.body);
-    } catch (err: any) {
-      if ("body" in err) {
-        logger.error(err.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  #throwUnlessNonNullableObject(
+    candidate: unknown
+  ): asserts candidate is NonNullable<Record<string, unknown>> {
+    if (typeof candidate !== "object" || candidate !== null) {
+      throw candidate;
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    this.#throwUnlessNonNullableObject(err);
+
+    if ("body" in err && err.body) {
+      logger.error(err.body);
+      this.#throwUnlessNonNullableObject(err.body);
+
+      if (typeof err.body.message === "string") {
+        throw new Error(err.body?.message);
       } else {
-        logger.error(err);
+        throw err.body;
       }
+    } else {
+      logger.error(err);
+      throw err;
     }
   }
 }
diff --git a/packages/core-apps/src/provider.ts b/packages/core-apps/src/provider.ts
index 28447d876..daeeea00d 100644
--- a/packages/core-apps/src/provider.ts
+++ b/packages/core-apps/src/provider.ts
@@ -100,13 +100,17 @@ export class ProviderShell implements Provider {
         },
         BACKGROUND_WORKER_MESSAGE: async (message) => {
           if (message.data.type === "SCHEDULE_ATTEMPT") {
-            this.tasks.create({
-              envId: message.data.envId,
-              runId: message.data.runId,
-              attemptId: message.data.id,
-              image: message.data.image,
-              machine: {},
-            });
+            try {
+              this.tasks.create({
+                envId: message.data.envId,
+                runId: message.data.runId,
+                attemptId: message.data.id,
+                image: message.data.image,
+                machine: {},
+              });
+            } catch (error) {
+              logger.error("create failed", error);
+            }
           }
         },
       },