Files
claude[bot] 69f396fbef fix(webapp): keep paused environments paused when concurrency limits are pushed (#4625)
<!-- ccr-slack-attribution -->
_Requested by **Matt Aitken** · [Slack
thread](https://triggerdotdev.slack.com/archives/C045W9WM3E1/p1786732623292829?thread_ts=1786732623.292829&cid=C045W9WM3E1)_

**Before:** you pause an environment, then a deploy lands (or a
background worker is created, or an admin changes the
concurrency/burst-factor). The environment starts picking up runs again
even though the dashboard still shows it as paused.

**After:** a paused environment stays paused until it is resumed, no
matter what else pushes its concurrency limit.

Pausing an environment sets `paused` in the database and writes a `0`
env concurrency limit into the run queue — the `0` is the only thing
that actually stops dequeueing. Any caller that pushed the limit without
an explicit value (`finalizeDeployment`, `createBackgroundWorker`, the
two admin environment routes) rewrote the real limit and silently
un-paused the environment.

##  Checklist

- [x] I have followed every step in the [contributing
guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md)
- [x] The PR title follows the convention.
- [x] I ran and tested the code works

---

## Testing

`apps/webapp/test/pauseEnvironment.server.test.ts` gains two
`containerTest` cases that wire a real `RunEngine` (real Redis) in place
of the stubbed app singleton and assert the actual run-queue env limit:

- pause a PRODUCTION env → limit is `0` → run the real
`FinalizeDeploymentService` → limit is still `0`, plus a control on a
running env in the same test proving that deploy path really does push
the limit (so the `0` can't just mean "nothing happened").
- pause → resume → the real limit is restored, so the clamp can't
regress resuming.

Both cases fail on `main` (`expected 17 to be +0` and `expected +0 to be
17`) and pass with this change. `pnpm run typecheck --filter webapp` is
clean.

---

## Changelog

Fix paused environments starting to run work again after a deploy.

---

## How

The clamp lives in the shared `updateEnvConcurrencyLimits` helper in
`apps/webapp/app/v3/runQueue.server.ts`, so every present and future
caller is covered: when no explicit limit is passed and the environment
is paused, `0` is written instead of the stored maximum. An
explicitly-passed limit still wins, which is what pausing itself relies
on. The resume path now passes the post-update environment state (its
in-memory copy was read before the un-pause and would otherwise be
clamped back to `0`), and the helper no longer mutates the caller's
environment object — that aliasing made a pause followed by a resume on
the same object write `0` twice. The existing `!paused` guards in
`allocateConcurrency` and the queue-level guard in
`createBackgroundWorker` are left in place as defence in depth, and
queue-level `TaskQueue.paused` behaviour is untouched.

---------

Co-authored-by: Claude <noreply@anthropic.com>
2026-08-14 22:12:25 +01:00

161 lines
5.1 KiB
TypeScript

import type { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { updateEnvConcurrencyLimits } from "../runQueue.server";
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
import { BaseService, ServiceValidationError } from "./baseService.server";
import { ChangeCurrentDeploymentService } from "./changeCurrentDeployment.server";
import { projectPubSub } from "./projectPubSub.server";
import { FailDeploymentService } from "./failDeployment.server";
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
import { DeploymentService } from "./deployment.server";
import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
import { engine } from "../runEngine.server";
import { tryCatch } from "@trigger.dev/core";
export class FinalizeDeploymentService extends BaseService {
public async call(
authenticatedEnv: AuthenticatedEnvironment,
id: string,
body: FinalizeDeploymentRequestBody
) {
const deployment = await this._prisma.workerDeployment.findFirst({
where: {
friendlyId: id,
environmentId: authenticatedEnv.id,
},
include: {
worker: {
include: {
tasks: true,
},
},
},
});
if (!deployment) {
logger.error("Worker deployment not found", { id });
return;
}
if (!deployment.worker) {
logger.error("Worker deployment does not have a worker", { id });
const failService = new FailDeploymentService();
await failService.call(authenticatedEnv, deployment.friendlyId, {
error: {
name: "MissingWorker",
message: "Deployment does not have a worker",
},
});
throw new ServiceValidationError("Worker deployment does not have a worker");
}
if (deployment.status === "DEPLOYED") {
logger.debug("Worker deployment is already deployed", { id });
return deployment;
}
if (deployment.status !== "DEPLOYING") {
logger.error("Worker deployment is not in DEPLOYING status", { id });
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
}
const imageDigest = validatedImageDigest(body.imageDigest);
// Link the deployment with the background worker
const finalizedDeployment = await this._prisma.workerDeployment.update({
where: {
id: deployment.id,
},
data: {
status: "DEPLOYED",
deployedAt: new Date(),
// Only add the digest, if any
imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined,
},
});
recordDeploymentOutcome({
status: "DEPLOYED",
deploymentFriendlyId: deployment.friendlyId,
organizationId: authenticatedEnv.organizationId,
projectId: authenticatedEnv.projectId,
environmentId: authenticatedEnv.id,
environmentType: authenticatedEnv.type,
});
const deploymentService = new DeploymentService();
await deploymentService
.appendToEventLog(authenticatedEnv.project, finalizedDeployment, [
{
type: "finalized",
data: {
result: "succeeded",
},
},
])
.orTee((error) => {
logger.error("Failed to append finalized deployment event to event log", { error });
});
await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
if (typeof body.skipPromotion === "undefined" || !body.skipPromotion) {
const promotionService = new ChangeCurrentDeploymentService();
await promotionService.call(finalizedDeployment, "promote");
}
try {
//send a notification that a new worker has been created
await projectPubSub.publish(
`project:${authenticatedEnv.projectId}:env:${authenticatedEnv.id}`,
"WORKER_CREATED",
{
environmentId: authenticatedEnv.id,
environmentType: authenticatedEnv.type,
createdAt: authenticatedEnv.createdAt,
taskCount: deployment.worker.tasks.length,
type: "deployed",
}
);
await updateEnvConcurrencyLimits(authenticatedEnv, undefined, this._prisma);
} catch (err) {
logger.error("Failed to publish WORKER_CREATED event", { err });
}
if (deployment.worker.engine === "V2") {
const [schedulePendingVersionsError] = await tryCatch(
engine.scheduleEnqueueRunsForBackgroundWorker(deployment.worker.id)
);
if (schedulePendingVersionsError) {
logger.error("Error scheduling pending versions", {
error: schedulePendingVersionsError,
});
}
}
await PerformDeploymentAlertsService.enqueue(deployment.id);
return finalizedDeployment;
}
}
function validatedImageDigest(imageDigest?: string): string | undefined {
if (!imageDigest) {
return;
}
if (!/^sha256:[a-f0-9]{64}$/.test(imageDigest.trim())) {
logger.error("Invalid image digest", { imageDigest });
return;
}
return imageDigest.trim();
}