Files
claude[bot] 69f396fbef fix(webapp): keep paused environments paused when concurrency limits are pushed (#4625)
<!-- ccr-slack-attribution -->
_Requested by **Matt Aitken** · [Slack
thread](https://triggerdotdev.slack.com/archives/C045W9WM3E1/p1786732623292829?thread_ts=1786732623.292829&cid=C045W9WM3E1)_

**Before:** you pause an environment, then a deploy lands (or a
background worker is created, or an admin changes the
concurrency/burst-factor). The environment starts picking up runs again
even though the dashboard still shows it as paused.

**After:** a paused environment stays paused until it is resumed, no
matter what else pushes its concurrency limit.

Pausing an environment sets `paused` in the database and writes a `0`
env concurrency limit into the run queue — the `0` is the only thing
that actually stops dequeueing. Any caller that pushed the limit without
an explicit value (`finalizeDeployment`, `createBackgroundWorker`, the
two admin environment routes) rewrote the real limit and silently
un-paused the environment.

##  Checklist

- [x] I have followed every step in the [contributing
guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md)
- [x] The PR title follows the convention.
- [x] I ran and tested the code works

---

## Testing

`apps/webapp/test/pauseEnvironment.server.test.ts` gains two
`containerTest` cases that wire a real `RunEngine` (real Redis) in place
of the stubbed app singleton and assert the actual run-queue env limit:

- pause a PRODUCTION env → limit is `0` → run the real
`FinalizeDeploymentService` → limit is still `0`, plus a control on a
running env in the same test proving that deploy path really does push
the limit (so the `0` can't just mean "nothing happened").
- pause → resume → the real limit is restored, so the clamp can't
regress resuming.

Both cases fail on `main` (`expected 17 to be +0` and `expected +0 to be
17`) and pass with this change. `pnpm run typecheck --filter webapp` is
clean.

---

## Changelog

Fix paused environments starting to run work again after a deploy.

---

## How

The clamp lives in the shared `updateEnvConcurrencyLimits` helper in
`apps/webapp/app/v3/runQueue.server.ts`, so every present and future
caller is covered: when no explicit limit is passed and the environment
is paused, `0` is written instead of the stored maximum. An
explicitly-passed limit still wins, which is what pausing itself relies
on. The resume path now passes the post-update environment state (its
in-memory copy was read before the un-pause and would otherwise be
clamped back to `0`), and the helper no longer mutates the caller's
environment object — that aliasing made a pause followed by a resume on
the same object write `0` twice. The existing `!paused` guards in
`allocateConcurrency` and the queue-level guard in
`createBackgroundWorker` are left in place as defence in depth, and
queue-level `TaskQueue.paused` behaviour is untouched.

---------

Co-authored-by: Claude <noreply@anthropic.com>
2026-08-14 22:12:25 +01:00

159 lines
5.1 KiB
TypeScript

import { EnvironmentPauseSource, type PrismaClientOrTransaction } from "@trigger.dev/database";
import { prisma } from "~/db.server";
import { logger } from "~/services/logger.server";
import { getManualPauseEnvironmentResult } from "~/v3/services/billingLimit/manualPauseEnvironmentGuard.server";
import { updateEnvConcurrencyLimits } from "../runQueue.server";
import { WithRunEngine } from "./baseService.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
export type PauseStatus = "paused" | "resumed";
export type PauseEnvironmentResult =
| {
success: true;
state: PauseStatus;
}
| {
success: false;
error: string;
};
export class PauseEnvironmentService extends WithRunEngine {
constructor(protected readonly _prisma: PrismaClientOrTransaction = prisma) {
super({ prisma });
}
public async call(
environment: AuthenticatedEnvironment,
action: PauseStatus
): Promise<PauseEnvironmentResult> {
try {
const org = await this._prisma.organization.findFirst({
where: {
id: environment.organizationId,
},
select: {
runsEnabled: true,
},
});
if (!org) {
throw new Error("Organization not found");
}
const previousPauseState = await this._prisma.runtimeEnvironment.findFirst({
where: { id: environment.id },
select: {
paused: true,
pauseSource: true,
},
});
const manualPauseGuard = getManualPauseEnvironmentResult(
action,
previousPauseState?.pauseSource
);
if (!manualPauseGuard.proceed) {
if (manualPauseGuard.success) {
return {
success: true,
state: manualPauseGuard.state,
};
}
// Expected, user-actionable guard result, not an error: return it as a failure
// result so it doesn't reach Sentry via the catch below.
return {
success: false,
error: manualPauseGuard.error,
};
}
if (!org.runsEnabled && action === "resumed") {
throw new Error(
"Runs are disabled for this organization. Your free plan has probably been exceeded. If not please contact support."
);
}
if (action === "resumed") {
const resumed = await this._prisma.runtimeEnvironment.updateMany({
where: {
id: environment.id,
// NOT on a nullable field excludes NULL rows in Prisma, which made
// user-paused envs (pauseSource null) unresumable.
OR: [
{ pauseSource: null },
{ NOT: { pauseSource: EnvironmentPauseSource.BILLING_LIMIT } },
],
},
data: {
paused: false,
pauseSource: null,
},
});
if (resumed.count === 0) {
// Raced into the paused state after the guard read above: expected,
// return as a failure result rather than throwing to Sentry.
return {
success: false,
error:
"This environment is paused because your organization reached its billing limit. Resolve the limit on the billing limits settings page to resume.",
};
}
} else {
await this._prisma.runtimeEnvironment.update({
where: { id: environment.id },
data: { paused: true },
});
}
try {
if (action === "paused") {
logger.debug("PauseEnvironmentService: pausing environment", {
environmentId: environment.id,
});
await updateEnvConcurrencyLimits(environment, 0);
} else {
logger.debug("PauseEnvironmentService: resuming environment", {
environmentId: environment.id,
});
// `environment` was read before the update above, so its `paused` is stale. The helper
// resolves the current state itself - hand it the client that wrote the resume.
await updateEnvConcurrencyLimits(environment, undefined, this._prisma);
}
} catch (error) {
await this._prisma.runtimeEnvironment.update({
where: { id: environment.id },
data: {
paused: previousPauseState?.paused ?? action === "resumed",
pauseSource: previousPauseState?.pauseSource ?? null,
},
});
// Rollback still wrote the env row; drop any cached copy before rethrowing.
controlPlaneResolver.invalidateEnvironment(environment.id);
throw error;
}
// The env's `paused` state changed in the control-plane; drop any cached copy.
controlPlaneResolver.invalidateEnvironment(environment.id);
return {
success: true,
state: action,
};
} catch (error) {
logger.error("PauseEnvironmentService: error pausing environment", {
action,
environmentId: environment.id,
error,
});
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error",
};
}
}
}