More tests passing, fixed the heartbeat issue

internal-packages/run-engine/src/engine/index.ts

@@ -1331,6 +1331,20 @@ export class RunEngine {
           }
 
           //it will automatically be requeued X times depending on the queue retry settings
+          await this.runAttemptSystem.tryNackAndRequeue({
+            run,
+            environment: {
+              id: latestSnapshot.environmentId,
+              type: latestSnapshot.environmentType,
+            },
+            orgId: run.runtimeEnvironment.organizationId,
+            error: {
+              type: "INTERNAL_ERROR",
+              code: "TASK_RUN_DEQUEUED_MAX_RETRIES",
+              message: `Trying to create an attempt failed multiple times, exceeding how many times we retry.`,
+            },
+            tx: prisma,
+          });
           break;
         }
         case "EXECUTING":

internal-packages/run-engine/src/engine/tests/heartbeats.test.ts

@@ -95,7 +95,7 @@ describe("RunEngine heartbeats", () => {
       assertNonNullable(executionData);
       expect(executionData.snapshot.executionStatus).toBe("PENDING_EXECUTING");
 
-      await setTimeout(pendingExecutingTimeout * 2);
+      await setTimeout(pendingExecutingTimeout * 4);
 
       //expect it to be pending with 3 consecutiveFailures
       const executionData2 = await engine.getRunExecutionData({ runId: run.id });

internal-packages/run-engine/src/engine/tests/locking.test.ts

@@ -1,7 +1,7 @@
 import { createRedisClient } from "@internal/redis";
 import { redisTest } from "@internal/testcontainers";
 import { expect } from "vitest";
-import { RunLocker } from "./locking.js";
+import { RunLocker } from "../locking.js";
 
 describe("RunLocker", () => {
   redisTest("Test acquiring a lock works", { timeout: 15_000 }, async ({ redisOptions }) => {

internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts

@@ -372,7 +372,6 @@ describe("RunEngine triggerAndWait", () => {
         const blockedResult = await engine.blockRunWithWaitpoint({
           runId: parentRun2.id,
           waitpoints: childRunWithWaitpoint.associatedWaitpoint!.id,
-          environmentId: authenticatedEnvironment.id,
           projectId: authenticatedEnvironment.project.id,
           organizationId: authenticatedEnvironment.organizationId,
           tx: prisma,

internal-packages/run-engine/src/run-queue/index.ts

@@ -473,7 +473,8 @@ export class RunQueue {
         const message = await this.readMessage(orgId, messageId);
 
         if (!message) {
-          throw new MessageNotFoundError(messageId);
+          // Message not found, it may have already been acknowledged
+          return;
         }
 
         span.setAttributes({