69f396fbef
<!-- ccr-slack-attribution --> _Requested by **Matt Aitken** · [Slack thread](https://triggerdotdev.slack.com/archives/C045W9WM3E1/p1786732623292829?thread_ts=1786732623.292829&cid=C045W9WM3E1)_ **Before:** you pause an environment, then a deploy lands (or a background worker is created, or an admin changes the concurrency/burst-factor). The environment starts picking up runs again even though the dashboard still shows it as paused. **After:** a paused environment stays paused until it is resumed, no matter what else pushes its concurrency limit. Pausing an environment sets `paused` in the database and writes a `0` env concurrency limit into the run queue — the `0` is the only thing that actually stops dequeueing. Any caller that pushed the limit without an explicit value (`finalizeDeployment`, `createBackgroundWorker`, the two admin environment routes) rewrote the real limit and silently un-paused the environment. ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [x] I ran and tested the code works --- ## Testing `apps/webapp/test/pauseEnvironment.server.test.ts` gains two `containerTest` cases that wire a real `RunEngine` (real Redis) in place of the stubbed app singleton and assert the actual run-queue env limit: - pause a PRODUCTION env → limit is `0` → run the real `FinalizeDeploymentService` → limit is still `0`, plus a control on a running env in the same test proving that deploy path really does push the limit (so the `0` can't just mean "nothing happened"). - pause → resume → the real limit is restored, so the clamp can't regress resuming. Both cases fail on `main` (`expected 17 to be +0` and `expected +0 to be 17`) and pass with this change. `pnpm run typecheck --filter webapp` is clean. --- ## Changelog Fix paused environments starting to run work again after a deploy. --- ## How The clamp lives in the shared `updateEnvConcurrencyLimits` helper in `apps/webapp/app/v3/runQueue.server.ts`, so every present and future caller is covered: when no explicit limit is passed and the environment is paused, `0` is written instead of the stored maximum. An explicitly-passed limit still wins, which is what pausing itself relies on. The resume path now passes the post-update environment state (its in-memory copy was read before the un-pause and would otherwise be clamped back to `0`), and the helper no longer mutates the caller's environment object — that aliasing made a pause followed by a resume on the same object write `0` twice. The existing `!paused` guards in `allocateConcurrency` and the queue-level guard in `createBackgroundWorker` are left in place as defence in depth, and queue-level `TaskQueue.paused` behaviour is untouched. --------- Co-authored-by: Claude <noreply@anthropic.com>
159 lines
5.1 KiB
TypeScript
159 lines
5.1 KiB
TypeScript
import { EnvironmentPauseSource, type PrismaClientOrTransaction } from "@trigger.dev/database";
|
|
import { prisma } from "~/db.server";
|
|
import { logger } from "~/services/logger.server";
|
|
import { getManualPauseEnvironmentResult } from "~/v3/services/billingLimit/manualPauseEnvironmentGuard.server";
|
|
import { updateEnvConcurrencyLimits } from "../runQueue.server";
|
|
import { WithRunEngine } from "./baseService.server";
|
|
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
|
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
|
|
|
export type PauseStatus = "paused" | "resumed";
|
|
|
|
export type PauseEnvironmentResult =
|
|
| {
|
|
success: true;
|
|
state: PauseStatus;
|
|
}
|
|
| {
|
|
success: false;
|
|
error: string;
|
|
};
|
|
|
|
export class PauseEnvironmentService extends WithRunEngine {
|
|
constructor(protected readonly _prisma: PrismaClientOrTransaction = prisma) {
|
|
super({ prisma });
|
|
}
|
|
|
|
public async call(
|
|
environment: AuthenticatedEnvironment,
|
|
action: PauseStatus
|
|
): Promise<PauseEnvironmentResult> {
|
|
try {
|
|
const org = await this._prisma.organization.findFirst({
|
|
where: {
|
|
id: environment.organizationId,
|
|
},
|
|
select: {
|
|
runsEnabled: true,
|
|
},
|
|
});
|
|
|
|
if (!org) {
|
|
throw new Error("Organization not found");
|
|
}
|
|
|
|
const previousPauseState = await this._prisma.runtimeEnvironment.findFirst({
|
|
where: { id: environment.id },
|
|
select: {
|
|
paused: true,
|
|
pauseSource: true,
|
|
},
|
|
});
|
|
|
|
const manualPauseGuard = getManualPauseEnvironmentResult(
|
|
action,
|
|
previousPauseState?.pauseSource
|
|
);
|
|
if (!manualPauseGuard.proceed) {
|
|
if (manualPauseGuard.success) {
|
|
return {
|
|
success: true,
|
|
state: manualPauseGuard.state,
|
|
};
|
|
}
|
|
// Expected, user-actionable guard result, not an error: return it as a failure
|
|
// result so it doesn't reach Sentry via the catch below.
|
|
return {
|
|
success: false,
|
|
error: manualPauseGuard.error,
|
|
};
|
|
}
|
|
|
|
if (!org.runsEnabled && action === "resumed") {
|
|
throw new Error(
|
|
"Runs are disabled for this organization. Your free plan has probably been exceeded. If not please contact support."
|
|
);
|
|
}
|
|
|
|
if (action === "resumed") {
|
|
const resumed = await this._prisma.runtimeEnvironment.updateMany({
|
|
where: {
|
|
id: environment.id,
|
|
// NOT on a nullable field excludes NULL rows in Prisma, which made
|
|
// user-paused envs (pauseSource null) unresumable.
|
|
OR: [
|
|
{ pauseSource: null },
|
|
{ NOT: { pauseSource: EnvironmentPauseSource.BILLING_LIMIT } },
|
|
],
|
|
},
|
|
data: {
|
|
paused: false,
|
|
pauseSource: null,
|
|
},
|
|
});
|
|
|
|
if (resumed.count === 0) {
|
|
// Raced into the paused state after the guard read above: expected,
|
|
// return as a failure result rather than throwing to Sentry.
|
|
return {
|
|
success: false,
|
|
error:
|
|
"This environment is paused because your organization reached its billing limit. Resolve the limit on the billing limits settings page to resume.",
|
|
};
|
|
}
|
|
} else {
|
|
await this._prisma.runtimeEnvironment.update({
|
|
where: { id: environment.id },
|
|
data: { paused: true },
|
|
});
|
|
}
|
|
|
|
try {
|
|
if (action === "paused") {
|
|
logger.debug("PauseEnvironmentService: pausing environment", {
|
|
environmentId: environment.id,
|
|
});
|
|
await updateEnvConcurrencyLimits(environment, 0);
|
|
} else {
|
|
logger.debug("PauseEnvironmentService: resuming environment", {
|
|
environmentId: environment.id,
|
|
});
|
|
// `environment` was read before the update above, so its `paused` is stale. The helper
|
|
// resolves the current state itself - hand it the client that wrote the resume.
|
|
await updateEnvConcurrencyLimits(environment, undefined, this._prisma);
|
|
}
|
|
} catch (error) {
|
|
await this._prisma.runtimeEnvironment.update({
|
|
where: { id: environment.id },
|
|
data: {
|
|
paused: previousPauseState?.paused ?? action === "resumed",
|
|
pauseSource: previousPauseState?.pauseSource ?? null,
|
|
},
|
|
});
|
|
// Rollback still wrote the env row; drop any cached copy before rethrowing.
|
|
controlPlaneResolver.invalidateEnvironment(environment.id);
|
|
throw error;
|
|
}
|
|
|
|
// The env's `paused` state changed in the control-plane; drop any cached copy.
|
|
controlPlaneResolver.invalidateEnvironment(environment.id);
|
|
|
|
return {
|
|
success: true,
|
|
state: action,
|
|
};
|
|
} catch (error) {
|
|
logger.error("PauseEnvironmentService: error pausing environment", {
|
|
action,
|
|
environmentId: environment.id,
|
|
error,
|
|
});
|
|
|
|
return {
|
|
success: false,
|
|
error: error instanceof Error ? error.message : "Unknown error",
|
|
};
|
|
}
|
|
}
|
|
}
|