69f396fbef
<!-- ccr-slack-attribution --> _Requested by **Matt Aitken** · [Slack thread](https://triggerdotdev.slack.com/archives/C045W9WM3E1/p1786732623292829?thread_ts=1786732623.292829&cid=C045W9WM3E1)_ **Before:** you pause an environment, then a deploy lands (or a background worker is created, or an admin changes the concurrency/burst-factor). The environment starts picking up runs again even though the dashboard still shows it as paused. **After:** a paused environment stays paused until it is resumed, no matter what else pushes its concurrency limit. Pausing an environment sets `paused` in the database and writes a `0` env concurrency limit into the run queue — the `0` is the only thing that actually stops dequeueing. Any caller that pushed the limit without an explicit value (`finalizeDeployment`, `createBackgroundWorker`, the two admin environment routes) rewrote the real limit and silently un-paused the environment. ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [x] I ran and tested the code works --- ## Testing `apps/webapp/test/pauseEnvironment.server.test.ts` gains two `containerTest` cases that wire a real `RunEngine` (real Redis) in place of the stubbed app singleton and assert the actual run-queue env limit: - pause a PRODUCTION env → limit is `0` → run the real `FinalizeDeploymentService` → limit is still `0`, plus a control on a running env in the same test proving that deploy path really does push the limit (so the `0` can't just mean "nothing happened"). - pause → resume → the real limit is restored, so the clamp can't regress resuming. Both cases fail on `main` (`expected 17 to be +0` and `expected +0 to be 17`) and pass with this change. `pnpm run typecheck --filter webapp` is clean. --- ## Changelog Fix paused environments starting to run work again after a deploy. --- ## How The clamp lives in the shared `updateEnvConcurrencyLimits` helper in `apps/webapp/app/v3/runQueue.server.ts`, so every present and future caller is covered: when no explicit limit is passed and the environment is paused, `0` is written instead of the stored maximum. An explicitly-passed limit still wins, which is what pausing itself relies on. The resume path now passes the post-update environment state (its in-memory copy was read before the un-pause and would otherwise be clamped back to `0`), and the helper no longer mutates the caller's environment object — that aliasing made a pause followed by a resume on the same object write `0` twice. The existing `!paused` guards in `allocateConcurrency` and the queue-level guard in `createBackgroundWorker` are left in place as defence in depth, and queue-level `TaskQueue.paused` behaviour is untouched. --------- Co-authored-by: Claude <noreply@anthropic.com>
161 lines
5.1 KiB
TypeScript
161 lines
5.1 KiB
TypeScript
import type { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
|
|
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
|
import { logger } from "~/services/logger.server";
|
|
import { updateEnvConcurrencyLimits } from "../runQueue.server";
|
|
import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
|
|
import { BaseService, ServiceValidationError } from "./baseService.server";
|
|
import { ChangeCurrentDeploymentService } from "./changeCurrentDeployment.server";
|
|
import { projectPubSub } from "./projectPubSub.server";
|
|
import { FailDeploymentService } from "./failDeployment.server";
|
|
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
|
|
import { DeploymentService } from "./deployment.server";
|
|
import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
|
|
import { engine } from "../runEngine.server";
|
|
import { tryCatch } from "@trigger.dev/core";
|
|
|
|
export class FinalizeDeploymentService extends BaseService {
|
|
public async call(
|
|
authenticatedEnv: AuthenticatedEnvironment,
|
|
id: string,
|
|
body: FinalizeDeploymentRequestBody
|
|
) {
|
|
const deployment = await this._prisma.workerDeployment.findFirst({
|
|
where: {
|
|
friendlyId: id,
|
|
environmentId: authenticatedEnv.id,
|
|
},
|
|
include: {
|
|
worker: {
|
|
include: {
|
|
tasks: true,
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
if (!deployment) {
|
|
logger.error("Worker deployment not found", { id });
|
|
return;
|
|
}
|
|
|
|
if (!deployment.worker) {
|
|
logger.error("Worker deployment does not have a worker", { id });
|
|
|
|
const failService = new FailDeploymentService();
|
|
await failService.call(authenticatedEnv, deployment.friendlyId, {
|
|
error: {
|
|
name: "MissingWorker",
|
|
message: "Deployment does not have a worker",
|
|
},
|
|
});
|
|
|
|
throw new ServiceValidationError("Worker deployment does not have a worker");
|
|
}
|
|
|
|
if (deployment.status === "DEPLOYED") {
|
|
logger.debug("Worker deployment is already deployed", { id });
|
|
|
|
return deployment;
|
|
}
|
|
|
|
if (deployment.status !== "DEPLOYING") {
|
|
logger.error("Worker deployment is not in DEPLOYING status", { id });
|
|
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
|
|
}
|
|
|
|
const imageDigest = validatedImageDigest(body.imageDigest);
|
|
|
|
// Link the deployment with the background worker
|
|
const finalizedDeployment = await this._prisma.workerDeployment.update({
|
|
where: {
|
|
id: deployment.id,
|
|
},
|
|
data: {
|
|
status: "DEPLOYED",
|
|
deployedAt: new Date(),
|
|
// Only add the digest, if any
|
|
imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined,
|
|
},
|
|
});
|
|
|
|
recordDeploymentOutcome({
|
|
status: "DEPLOYED",
|
|
deploymentFriendlyId: deployment.friendlyId,
|
|
organizationId: authenticatedEnv.organizationId,
|
|
projectId: authenticatedEnv.projectId,
|
|
environmentId: authenticatedEnv.id,
|
|
environmentType: authenticatedEnv.type,
|
|
});
|
|
|
|
const deploymentService = new DeploymentService();
|
|
await deploymentService
|
|
.appendToEventLog(authenticatedEnv.project, finalizedDeployment, [
|
|
{
|
|
type: "finalized",
|
|
data: {
|
|
result: "succeeded",
|
|
},
|
|
},
|
|
])
|
|
.orTee((error) => {
|
|
logger.error("Failed to append finalized deployment event to event log", { error });
|
|
});
|
|
|
|
await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
|
|
|
|
if (typeof body.skipPromotion === "undefined" || !body.skipPromotion) {
|
|
const promotionService = new ChangeCurrentDeploymentService();
|
|
|
|
await promotionService.call(finalizedDeployment, "promote");
|
|
}
|
|
|
|
try {
|
|
//send a notification that a new worker has been created
|
|
await projectPubSub.publish(
|
|
`project:${authenticatedEnv.projectId}:env:${authenticatedEnv.id}`,
|
|
"WORKER_CREATED",
|
|
{
|
|
environmentId: authenticatedEnv.id,
|
|
environmentType: authenticatedEnv.type,
|
|
createdAt: authenticatedEnv.createdAt,
|
|
taskCount: deployment.worker.tasks.length,
|
|
type: "deployed",
|
|
}
|
|
);
|
|
|
|
await updateEnvConcurrencyLimits(authenticatedEnv, undefined, this._prisma);
|
|
} catch (err) {
|
|
logger.error("Failed to publish WORKER_CREATED event", { err });
|
|
}
|
|
|
|
if (deployment.worker.engine === "V2") {
|
|
const [schedulePendingVersionsError] = await tryCatch(
|
|
engine.scheduleEnqueueRunsForBackgroundWorker(deployment.worker.id)
|
|
);
|
|
|
|
if (schedulePendingVersionsError) {
|
|
logger.error("Error scheduling pending versions", {
|
|
error: schedulePendingVersionsError,
|
|
});
|
|
}
|
|
}
|
|
|
|
await PerformDeploymentAlertsService.enqueue(deployment.id);
|
|
|
|
return finalizedDeployment;
|
|
}
|
|
}
|
|
|
|
function validatedImageDigest(imageDigest?: string): string | undefined {
|
|
if (!imageDigest) {
|
|
return;
|
|
}
|
|
|
|
if (!/^sha256:[a-f0-9]{64}$/.test(imageDigest.trim())) {
|
|
logger.error("Invalid image digest", { imageDigest });
|
|
return;
|
|
}
|
|
|
|
return imageDigest.trim();
|
|
}
|