From 19908436b84181e5dc76ab775b9d17634940566a Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Thu, 20 Aug 2026 07:08:22 +0100 Subject: [PATCH] perf(ci): speed up webapp test execution (#4709) ## Summary Speeds up webapp test jobs by balancing measured work across runners, reducing repeated container setup, and ensuring test workers release shutdown resources promptly. Unit tests run across 24 duration-aware shards, while E2E tests run across two balanced shards. ## Design `RunEngine` shutdown now closes processing resources before support resources, continues cleanup if one close fails, and reuses one shutdown promise for concurrent callers. Redis workers clear completed shutdown deadlines so finished tests no longer wait on idle timers. Container-heavy suites are split only where it improves parallelism, and repeated replication and engine fixtures are consolidated where one end-to-end case provides coverage. Timing weights are refreshed for all affected files. Dependency installation overlaps container pulls, and both workflows use WarpBuild's Node setup action. --- .github/workflows/e2e-webapp.yml | 65 +- .github/workflows/unit-tests-webapp.yml | 66 +- .../triggerTask.server.nullBytes.test.ts | 37 +- .../apiRunListPresenter.readthrough.test.ts | 158 + apps/webapp/test/apiRunListPresenter.test.ts | 393 +- .../test/dashboardAgentWatches.batch.test.ts | 704 ++++ .../dashboardAgentWatches.delivery.test.ts | 1368 +++++++ .../dashboardAgentWatches.lifecycle.test.ts | 1302 +++++++ .../webapp/test/dashboardAgentWatches.test.ts | 3421 ----------------- ...riggerTask.externalDeploymentId.helpers.ts | 117 + ...rTask.externalDeploymentId.pending.test.ts | 131 + ...sk.externalDeploymentId.resolution.test.ts | 199 + .../triggerTask.externalDeploymentId.test.ts | 354 +- .../envConcurrencyLimitPause.server.test.ts | 151 +- ...ConcurrencyLimitPauseDirect.server.test.ts | 71 + ...oncurrencyLimitPauseService.server.test.ts | 56 + .../helpers/apiRunListPresenterTestHelpers.ts | 178 + .../dashboardAgentWatchesTestHelpers.ts | 219 ++ .../envConcurrencyLimitPauseTestHelpers.ts | 94 + apps/webapp/vitest.e2e.config.ts | 2 + .../run-engine/src/engine/index.ts | 78 +- .../src/engine/tests/shutdown.test.ts | 94 + .../src/fair-queue/tests/fairQueue.test.ts | 27 + .../src/fair-queue/workerQueue.ts | 8 +- packages/redis-worker/src/worker.test.ts | 69 +- packages/redis-worker/src/worker.ts | 22 +- test-timings.json | 805 ++-- 27 files changed, 5598 insertions(+), 4591 deletions(-) create mode 100644 apps/webapp/test/apiRunListPresenter.readthrough.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.batch.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.delivery.test.ts create mode 100644 apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts delete mode 100644 apps/webapp/test/dashboardAgentWatches.test.ts create mode 100644 apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts create mode 100644 apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts create mode 100644 apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts create mode 100644 apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts create mode 100644 apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts create mode 100644 apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts create mode 100644 apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts create mode 100644 apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts create mode 100644 internal-packages/run-engine/src/engine/tests/shutdown.test.ts diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml index a05b4525d..e8f778d75 100644 --- a/.github/workflows/e2e-webapp.yml +++ b/.github/workflows/e2e-webapp.yml @@ -16,8 +16,15 @@ jobs: name: "๐Ÿงช E2E Tests: Webapp" runs-on: warp-ubuntu-latest-x64-16x timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2] + shardTotal: [2] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + SHARD_INDEX: ${{ matrix.shardIndex }} + SHARD_TOTAL: ${{ matrix.shardTotal }} steps: - name: ๐Ÿ”ง Disable IPv6 run: | @@ -57,7 +64,7 @@ jobs: version: 10.33.2 - name: โŽ” Setup node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 with: node-version: 24.18.0 cache: "pnpm" @@ -73,18 +80,52 @@ jobs: if: ${{ !env.DOCKERHUB_USERNAME }} run: echo "DockerHub login skipped because secrets are not available." - - name: ๐Ÿณ Pre-pull testcontainer images + - name: ๐Ÿ“ฅ Prepare deps and testcontainer images run: | - echo "Pre-pulling Docker images with authenticated session..." - docker pull postgres:14 - docker pull redis:7.2 - docker pull testcontainers/ryuk:0.14.0 - docker pull ghcr.io/s2-streamstore/s2:0.40.0@sha256:b26249e2ede0949755f5af8028185dc2bcfc3aa2db21eb9610543d144eb6ee9d - docker pull minio/minio:latest - echo "Image pre-pull complete" + # Pull images concurrently with dependency installation. Retry each pull because + # registry timeouts are a recurring transient CI flake. + pull() { + for attempt in 1 2 3; do + docker pull "$1" && return 0 + echo "::warning::docker pull $1 failed (attempt ${attempt}/3); retrying in 10s" + sleep 10 + done + echo "::error::docker pull $1 failed after 3 attempts" + return 1 + } - - name: ๐Ÿ“ฅ Download deps - run: pnpm install --frozen-lockfile + pull_images() { + local pids=() + local failed=0 + for image in \ + postgres:14 \ + redis:7.2 \ + testcontainers/ryuk:0.14.0 \ + ghcr.io/s2-streamstore/s2:0.40.0@sha256:b26249e2ede0949755f5af8028185dc2bcfc3aa2db21eb9610543d144eb6ee9d \ + minio/minio:latest + do + pull "$image" & + pids+=("$!") + done + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + failed=1 + fi + done + return "$failed" + } + + echo "Installing dependencies and pre-pulling Docker images..." + pull_images & + pull_pid=$! + install_status=0 + pnpm install --frozen-lockfile || install_status=$? + pull_status=0 + wait "$pull_pid" || pull_status=$? + if (( install_status != 0 || pull_status != 0 )); then + exit 1 + fi + echo "Dependency install and image pre-pull complete" - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate @@ -96,6 +137,6 @@ jobs: run: cd apps/webapp && pnpm exec playwright install chromium - name: ๐Ÿงช Run Webapp E2E Tests - run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default + run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} env: WEBAPP_TEST_VERBOSE: "1" diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index bec77bbbc..0a3fa28b6 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -14,18 +14,18 @@ on: jobs: unitTests: name: "๐Ÿงช Unit Tests: Webapp" - # 10 shards on 16x machines: webapp test throughput is limited per-machine (one - # docker daemon + disk absorbing all the per-file Postgres/ClickHouse container - # spin-up), so many machines beats few big ones - fewer/bigger (3x32) measured - # SLOWER than 10x8. The 16x (vs 8x) gives the fork pool the CPU headroom the 8x - # runners lacked. Setup overhead per machine is ~1 min on warm runners. + # Webapp test throughput is limited per-machine (one docker daemon + disk absorbing + # all the per-file Postgres/ClickHouse container spin-up), so many machines beats + # few big ones - fewer/bigger (3x32) measured slower than 10x8. The 16x (vs 8x) + # gives the fork pool the CPU headroom the 8x runners lacked. runs-on: warp-ubuntu-latest-x64-16x strategy: # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard fail-fast: false matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - shardTotal: [12] + shardIndex: + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + shardTotal: [24] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} @@ -69,7 +69,7 @@ jobs: version: 10.33.2 - name: โŽ” Setup node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + uses: WarpBuilds/setup-node@bc639b444d583175926b588962199c247d23e8d3 # v6 with: node-version: 24.18.0 cache: "pnpm" @@ -85,9 +85,10 @@ jobs: if: ${{ !env.DOCKERHUB_USERNAME }} run: echo "DockerHub login skipped because secrets are not available." - - name: ๐Ÿณ Pre-pull testcontainer images + - name: ๐Ÿ“ฅ Prepare deps and testcontainer images run: | - # Retry each pull - DockerHub registry timeouts are a recurring transient CI flake. + # Pull images concurrently with dependency installation. Retry each pull because + # DockerHub registry timeouts are a recurring transient CI flake. pull() { for attempt in 1 2 3; do docker pull "$1" && return 0 @@ -97,18 +98,41 @@ jobs: echo "::error::docker pull $1 failed after 3 attempts" return 1 } - echo "Pre-pulling Docker images with authenticated session..." - pull postgres:14 - pull postgres:17 - pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251 - pull redis:7.2 - pull testcontainers/ryuk:0.14.0 - pull electricsql/electric:1.2.4@sha256:20da3d0b0e74926c5623392db67fd56698b9e374c4aeb6cb5cadeb8fea171c36 - pull minio/minio:latest - echo "Image pre-pull complete" - - name: ๐Ÿ“ฅ Download deps - run: pnpm install --frozen-lockfile + pull_images() { + local pids=() + local failed=0 + for image in \ + postgres:14 \ + postgres:17 \ + clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251 \ + redis:7.2 \ + testcontainers/ryuk:0.14.0 \ + electricsql/electric:1.2.4@sha256:20da3d0b0e74926c5623392db67fd56698b9e374c4aeb6cb5cadeb8fea171c36 \ + minio/minio:latest + do + pull "$image" & + pids+=("$!") + done + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + failed=1 + fi + done + return "$failed" + } + + echo "Installing dependencies and pre-pulling Docker images..." + pull_images & + pull_pid=$! + install_status=0 + pnpm install --frozen-lockfile || install_status=$? + pull_status=0 + wait "$pull_pid" || pull_status=$? + if (( install_status != 0 || pull_status != 0 )); then + exit 1 + fi + echo "Dependency install and image pre-pull complete" - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts index 9612f103e..29bb51ff3 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts @@ -50,7 +50,7 @@ function buildService(engine: any, prisma: any) { describe("RunEngineTriggerTaskService null-byte sanitization", () => { containerTest( - "strips a NUL from idempotencyKeyOptions.key so the jsonb insert does not 22P05", + "sanitizes NUL-containing idempotency and debounce keys before the jsonb insert", async ({ prisma, redisOptions }) => { const engine = buildEngine(prisma, redisOptions); @@ -59,41 +59,13 @@ describe("RunEngineTriggerTaskService null-byte sanitization", () => { const service = buildService(engine, prisma); const result = await service.call({ - taskId: "nul-idem-task", + taskId: "nul-keys-task", environment, body: { - payload: { kind: "idem" }, + payload: { kind: "nul-keys" }, options: { idempotencyKey: "a".repeat(64), idempotencyKeyOptions: { key: `acme${NUL}inc`, scope: "run" }, - }, - }, - }); - assertNonNullable(result); - - const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } }); - expect(row.idempotencyKeyOptions).toEqual({ key: "acmeinc", scope: "run" }); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "strips a NUL from debounce.key so the jsonb insert does not 22P05", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const service = buildService(engine, prisma); - - const result = await service.call({ - taskId: "nul-debounce-task", - environment, - body: { - payload: { kind: "debounce" }, - options: { debounce: { key: `grp${NUL}1`, delay: "1s" }, }, }, @@ -101,7 +73,8 @@ describe("RunEngineTriggerTaskService null-byte sanitization", () => { assertNonNullable(result); const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } }); - expect((row.debounce as { key: string }).key).toBe("grp1"); + expect(row.idempotencyKeyOptions).toEqual({ key: "acmeinc", scope: "run" }); + expect(row.debounce).toMatchObject({ key: "grp1", delay: "1s" }); } finally { await engine.quit(); } diff --git a/apps/webapp/test/apiRunListPresenter.readthrough.test.ts b/apps/webapp/test/apiRunListPresenter.readthrough.test.ts new file mode 100644 index 000000000..84809630f --- /dev/null +++ b/apps/webapp/test/apiRunListPresenter.readthrough.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, vi } from "vitest"; + +// The presenter graph imports `~/db.server` singletons even though the asserted reads use explicit +// clients. These lazy proxies delegate every access to the real per-test Postgres containers. +const legacyReplicaHolder = vi.hoisted(() => ({ client: undefined as any })); +const newClientHolder = vi.hoisted(() => ({ client: undefined as any })); +const clickhouseHolder = vi.hoisted(() => ({ client: undefined as any })); + +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { + getClickhouseForOrganization: async () => { + if (!clickhouseHolder.client) { + throw new Error("clickhouseHolder.client not set for this test"); + } + return clickhouseHolder.client; + }, + }, +})); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_target, property) { + if (!holder.client) { + throw new Error(`${label} not set for this test`); + } + return holder.client[property]; + }, + } + ); + const replicaProxy = lazyProxy(legacyReplicaHolder, "legacyReplicaHolder.client"); + const newProxy = lazyProxy(newClientHolder, "newClientHolder.client"); + + return { + prisma: replicaProxy, + $replica: replicaProxy, + runOpsNewPrisma: newProxy, + runOpsNewReplica: newProxy, + runOpsLegacyPrisma: replicaProxy, + runOpsLegacyReplica: replicaProxy, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +import { createPostgresContainer, replicationContainerTest } from "@internal/testcontainers"; +import { PrismaClient } from "@trigger.dev/database"; +import { z } from "zod"; +import { CURRENT_API_VERSION } from "~/api/versions"; +import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server"; +import { createRun, mirrorParents, seedParents } from "./helpers/apiRunListPresenterTestHelpers"; +import { setupClickhouseReplication } from "./utils/replicationUtils"; + +vi.setConfig({ testTimeout: 90_000 }); + +describe("ApiRunListPresenter public /runs routed read-through", () => { + replicationContainerTest( + "public payload lists run-ops rows served via the routed store (NEW + legacy union)", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma, network }) => { + const { clickhouse } = await setupClickhouseReplication({ + prisma, + databaseUrl: postgresContainer.getConnectionUri(), + clickhouseUrl: clickhouseContainer.getConnectionUrl(), + redisOptions, + }); + + const { url: newUrl } = await createPostgresContainer(network, { + imageTag: "docker.io/postgres:17", + }); + const prismaNew = new PrismaClient({ datasources: { db: { url: newUrl } } }); + legacyReplicaHolder.client = prisma; + clickhouseHolder.client = clickhouse; + newClientHolder.client = prismaNew; + + try { + const ctx = await seedParents(prisma, "hydrate"); + await mirrorParents(prismaNew, ctx, "hydrate"); + + // PG14 is the logical-replication source, so ClickHouse receives the complete ID set. + const legacyOnlyA = await createRun(prisma, ctx, { friendlyId: "run_legacyA" }); + const legacyOnlyB = await createRun(prisma, ctx, { friendlyId: "run_legacyB" }); + const migratedA = await createRun(prisma, ctx, { friendlyId: "run_newA" }); + const migratedB = await createRun(prisma, ctx, { friendlyId: "run_newB" }); + + // The routed PG17 rows use distinguishing values that prove NEW hydration won. + await createRun(prismaNew, ctx, { + friendlyId: "run_newA", + taskIdentifier: "my-task-NEW", + }); + await createRun(prismaNew, ctx, { + friendlyId: "run_newB", + taskIdentifier: "my-task-NEW", + }); + await prismaNew.taskRun.update({ + where: { friendlyId: "run_newA" }, + data: { id: migratedA.id }, + }); + await prismaNew.taskRun.update({ + where: { friendlyId: "run_newB" }, + data: { id: migratedB.id }, + }); + + const replicatedRunsQuery = clickhouse.reader.query({ + name: "waitForApiRunListPresenterTaskRuns", + query: + "SELECT countDistinct(run_id) AS count FROM trigger_dev.task_runs_v2 WHERE run_id IN {run_ids:Array(String)}", + schema: z.object({ count: z.number() }), + params: z.object({ run_ids: z.array(z.string()) }), + }); + await vi.waitFor( + async () => { + const [error, rows] = await replicatedRunsQuery({ + run_ids: [legacyOnlyA.id, legacyOnlyB.id, migratedA.id, migratedB.id], + }); + if (error) throw error; + expect(rows?.[0]?.count).toBe(4); + }, + { timeout: 15_000, interval: 100 } + ); + + const presenter = new ApiRunListPresenter(prisma, prisma, { + newClient: prismaNew, + legacyReplica: prisma, + splitEnabled: true, + }); + + const result = await presenter.call( + { id: ctx.projectId }, + { "page[size]": 10 } as any, + CURRENT_API_VERSION, + { id: ctx.environmentId, organizationId: ctx.organizationId } + ); + + const expectedFriendlyIds = [ + { id: migratedA.id, friendlyId: "run_newA" }, + { id: migratedB.id, friendlyId: "run_newB" }, + { id: legacyOnlyA.id, friendlyId: "run_legacyA" }, + { id: legacyOnlyB.id, friendlyId: "run_legacyB" }, + ] + .sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)) + .map((run) => run.friendlyId); + expect(result.data.map((run) => run.id)).toEqual(expectedFriendlyIds); + + const migratedRow = result.data.find((run) => run.id === "run_newA"); + expect(migratedRow?.taskIdentifier).toBe("my-task-NEW"); + expect(migratedRow?.taskKind).toBe("STANDARD"); + expect(result.data.find((run) => run.id === "run_legacyA")?.taskIdentifier).toBe("my-task"); + + expect(result.pagination).toHaveProperty("next"); + expect(result.pagination).toHaveProperty("previous"); + } finally { + await prismaNew.$disconnect(); + } + } + ); +}); diff --git a/apps/webapp/test/apiRunListPresenter.test.ts b/apps/webapp/test/apiRunListPresenter.test.ts index bc9826461..a9f04ae52 100644 --- a/apps/webapp/test/apiRunListPresenter.test.ts +++ b/apps/webapp/test/apiRunListPresenter.test.ts @@ -1,20 +1,11 @@ import { describe, expect, vi } from "vitest"; -// The presenter graph imports `~/v3/runStore.server` (via RunsRepository) which imports -// `~/db.server` at load, and the presenter itself reaches `~/db.server`'s `$replica` singleton -// through `findDisplayableEnvironment` and `getTaskIdentifiers`. Stub the module so those -// singleton reads resolve. This is the ONLY mock โ€” the DB is NEVER mocked; the proxy delegates -// to the per-test REAL legacy (PG14) container so the env-lookup + task-identifier reads hit a -// real database. Everything asserted runs against real containers. Mirrors -// nextRunListPresenter.readthrough.test.ts. +// The presenter graph imports `~/db.server` singletons even though these tests pass explicit real +// clients. The proxies keep those singleton reads on the current warm Postgres fixture. const legacyReplicaHolder = vi.hoisted(() => ({ client: undefined as any })); const newClientHolder = vi.hoisted(() => ({ client: undefined as any })); -// `ApiRunListPresenter` resolves its read ClickHouse internally via the `clickhouseFactory` -// singleton (which imports `~/env.server` and binds to a process-wide default client). Stub the -// instance module so `getClickhouseForOrganization` returns the per-test container's ClickHouse -// handle (set by each test before calling). This is a module-resolution shim โ€” the ClickHouse is -// a REAL testcontainer, never mocked โ€” mirroring the `~/db.server` stub below. const clickhouseHolder = vi.hoisted(() => ({ client: undefined as any })); + vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ clickhouseFactory: { getClickhouseForOrganization: async () => { @@ -25,22 +16,24 @@ vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ }, }, })); + vi.mock("~/db.server", async () => { const { Prisma } = await import("@trigger.dev/database"); const lazyProxy = (holder: { client: any }, label: string) => new Proxy( {}, { - get(_t, prop) { + get(_target, property) { if (!holder.client) { throw new Error(`${label} not set for this test`); } - return holder.client[prop]; + return holder.client[property]; }, } ); const replicaProxy = lazyProxy(legacyReplicaHolder, "legacyReplicaHolder.client"); const newProxy = lazyProxy(newClientHolder, "newClientHolder.client"); + return { prisma: replicaProxy, $replica: replicaProxy, @@ -52,347 +45,101 @@ vi.mock("~/db.server", async () => { }; }); -import { createPostgresContainer, replicationContainerTest } from "@internal/testcontainers"; -import { PrismaClient } from "@trigger.dev/database"; -import { setTimeout } from "node:timers/promises"; +import { ClickHouse } from "@internal/clickhouse"; +import { containerTest } from "@internal/testcontainers"; import { CURRENT_API_VERSION } from "~/api/versions"; import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server"; -import { setupClickhouseReplication } from "./utils/replicationUtils"; +import { + addEnvironment, + createRun, + insertTaskRunV2Rows, + seedParents, +} from "./helpers/apiRunListPresenterTestHelpers"; vi.setConfig({ testTimeout: 90_000 }); -type SeedContext = { - organizationId: string; - projectId: string; - environmentId: string; - environmentSlug: string; -}; - -/** - * Creates the org/project/env parents on a single prisma client. TaskRun FKs require these to - * exist on every DB a run lives on, so identical parents (same ids) are seeded on both the - * legacy (PG14) and new (PG17) databases. - */ -async function seedParents( - prisma: PrismaClient, - slug: string, - envSlug = `env-${slug}` -): Promise { - const organization = await prisma.organization.create({ - data: { title: `org-${slug}`, slug: `org-${slug}` }, +function setupClients(prisma: unknown, clickhouseUrl: string): ClickHouse { + const clickhouse = new ClickHouse({ + url: clickhouseUrl, + name: "api-run-list-presenter-test", + compression: { request: true }, }); - const project = await prisma.project.create({ - data: { - name: `proj-${slug}`, - slug: `proj-${slug}`, - organizationId: organization.id, - externalRef: `proj-${slug}`, - }, - }); - const runtimeEnvironment = await prisma.runtimeEnvironment.create({ - data: { - slug: envSlug, - type: "DEVELOPMENT", - projectId: project.id, - organizationId: organization.id, - apiKey: `tr_dev_${slug}`, - pkApiKey: `pk_dev_${slug}`, - shortcode: `sc-${slug}`, - }, - }); - - return { - organizationId: organization.id, - projectId: project.id, - environmentId: runtimeEnvironment.id, - environmentSlug: runtimeEnvironment.slug, - }; + legacyReplicaHolder.client = prisma; + newClientHolder.client = prisma; + clickhouseHolder.client = clickhouse; + return clickhouse; } -/** Adds an extra RuntimeEnvironment (control-plane row) to an existing project. */ -async function addEnvironment( - prisma: PrismaClient, - ctx: SeedContext, - slug: string, - envSlug: string -): Promise { - const env = await prisma.runtimeEnvironment.create({ - data: { - slug: envSlug, - type: "STAGING", - projectId: ctx.projectId, - organizationId: ctx.organizationId, - apiKey: `tr_${envSlug}_${slug}`, - pkApiKey: `pk_${envSlug}_${slug}`, - shortcode: `sc-${envSlug}-${slug}`, - }, - }); - return env.id; -} - -/** Mirrors the org/project/env parents onto a second DB with the SAME ids. */ -async function mirrorParents(prisma: PrismaClient, ctx: SeedContext, slug: string): Promise { - await prisma.organization.create({ - data: { id: ctx.organizationId, title: `org-${slug}`, slug: `org-${slug}` }, - }); - await prisma.project.create({ - data: { - id: ctx.projectId, - name: `proj-${slug}`, - slug: `proj-${slug}`, - organizationId: ctx.organizationId, - externalRef: `proj-${slug}`, - }, - }); - await prisma.runtimeEnvironment.create({ - data: { - id: ctx.environmentId, - slug: ctx.environmentSlug, - type: "DEVELOPMENT", - projectId: ctx.projectId, - organizationId: ctx.organizationId, - apiKey: `tr_dev_${slug}_b`, - pkApiKey: `pk_dev_${slug}_b`, - shortcode: `sc-${slug}-b`, - }, - }); -} - -async function createRun( - prisma: PrismaClient, - ctx: SeedContext, - run: { - friendlyId: string; - taskIdentifier?: string; - status?: any; - runtimeEnvironmentId?: string; - } -) { - return prisma.taskRun.create({ - data: { - friendlyId: run.friendlyId, - taskIdentifier: run.taskIdentifier ?? "my-task", - status: run.status ?? "PENDING", - payload: JSON.stringify({ foo: run.friendlyId }), - traceId: run.friendlyId, - spanId: run.friendlyId, - queue: "test", - runTags: [], - runtimeEnvironmentId: run.runtimeEnvironmentId ?? ctx.environmentId, - projectId: ctx.projectId, - organizationId: ctx.organizationId, - environmentType: "DEVELOPMENT", - engine: "V2", - }, - }); -} - -describe("ApiRunListPresenter public /runs list (PG14 legacy + PG17 new)", () => { - // Public list serves run-ops rows through the routed store. The - // forwarded readThroughDeps thread the dual-DB union into NextRunListPresenter; the public - // payload (`{ data, pagination }`) must list the NEW โˆช legacy union, proving the public API - // surfaces routed run-ops rows. The migrated/straggler rows (run_newA/run_newB) live on BOTH - // DBs with the same id + friendlyId but a DISTINGUISHING taskIdentifier ("my-task-NEW" on PG17), - // so a row served from the threaded newClient is identifiable in the public payload. - replicationContainerTest( - "public payload lists run-ops rows served via the routed store (NEW + legacy union)", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma, network }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, - }); - - const { url: newUrl } = await createPostgresContainer(network, { - imageTag: "docker.io/postgres:17", - }); - const prismaNew = new PrismaClient({ datasources: { db: { url: newUrl } } }); - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; - // The routed store's default known-migrated probe reads `runOpsNewPrisma` -> PG17. - newClientHolder.client = prismaNew; - - try { - const ctx = await seedParents(prisma, "hydrate"); - await mirrorParents(prismaNew, ctx, "hydrate"); - - // All four runs land on PG14 (legacy + replication source -> CH gets the full id-set). - const legacyOnlyA = await createRun(prisma, ctx, { friendlyId: "run_legacyA" }); - const legacyOnlyB = await createRun(prisma, ctx, { friendlyId: "run_legacyB" }); - const migratedA = await createRun(prisma, ctx, { friendlyId: "run_newA" }); - const migratedB = await createRun(prisma, ctx, { friendlyId: "run_newB" }); - - // The two "migrated" runs also live on NEW (authoritative during retention), same ids + - // friendlyIds, but a DISTINGUISHING taskIdentifier so a row served from PG17 is - // identifiable in the public payload. - await createRun(prismaNew, ctx, { friendlyId: "run_newA", taskIdentifier: "my-task-NEW" }); - await createRun(prismaNew, ctx, { friendlyId: "run_newB", taskIdentifier: "my-task-NEW" }); - await prismaNew.taskRun.update({ - where: { friendlyId: "run_newA" }, - data: { id: migratedA.id }, - }); - await prismaNew.taskRun.update({ - where: { friendlyId: "run_newB" }, - data: { id: migratedB.id }, - }); - - // Wait for CH replication so the id-set page is non-empty. - await setTimeout(1500); - - const presenter = new ApiRunListPresenter(prisma, prisma, { - newClient: prismaNew, - legacyReplica: prisma, - splitEnabled: true, - }); - - const result = await presenter.call( - { id: ctx.projectId }, - { "page[size]": 10 } as any, - CURRENT_API_VERSION, - { id: ctx.environmentId, organizationId: ctx.organizationId } - ); - - // The public payload lists runs by `id` = `run.friendlyId`, id-desc ordered. - const expectedFriendlyIds = [ - { id: migratedA.id, friendlyId: "run_newA" }, - { id: migratedB.id, friendlyId: "run_newB" }, - { id: legacyOnlyA.id, friendlyId: "run_legacyA" }, - { id: legacyOnlyB.id, friendlyId: "run_legacyB" }, - ] - .sort((a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0)) - .map((r) => r.friendlyId); - expect(result.data.map((r) => r.id)).toEqual(expectedFriendlyIds); - - // The migrated rows must carry the PG17-only taskIdentifier โ€” only possible if the public - // path hydrated them through the threaded newClient (PG17). taskKind falls back to STANDARD. - const migratedRow = result.data.find((r) => r.id === "run_newA"); - expect(migratedRow?.taskIdentifier).toBe("my-task-NEW"); - expect(migratedRow?.taskKind).toBe("STANDARD"); - // The legacy-only rows surface from PG14, proving the legacyReplica is also exercised. - expect(result.data.find((r) => r.id === "run_legacyA")?.taskIdentifier).toBe("my-task"); - - // Pagination shape is present. - expect(result.pagination).toHaveProperty("next"); - expect(result.pagination).toHaveProperty("previous"); - } finally { - await prismaNew.$disconnect(); - } - } - ); - - // Genuinely-empty env returns { data: [], pagination } without error. Exercises the - // empty-state probe beneath NextRunListPresenter (no rows on either DB; empty CH page). - replicationContainerTest( +describe("ApiRunListPresenter public /runs list", () => { + containerTest( "genuinely-empty env returns { data: [], pagination } without error", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma, network }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, + async ({ clickhouseContainer, prisma }) => { + setupClients(prisma, clickhouseContainer.getConnectionUrl()); + const ctx = await seedParents(prisma, "empty"); + + // Keep the split/read-through branch active while both real clients point at the empty DB. + const presenter = new ApiRunListPresenter(prisma, prisma, { + newClient: prisma, + legacyReplica: prisma, + splitEnabled: true, }); - const { url: newUrl } = await createPostgresContainer(network, { - imageTag: "docker.io/postgres:17", - }); - const prismaNew = new PrismaClient({ datasources: { db: { url: newUrl } } }); - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; + const result = await presenter.call( + { id: ctx.projectId }, + { "page[size]": 10 } as any, + CURRENT_API_VERSION, + { id: ctx.environmentId, organizationId: ctx.organizationId } + ); - try { - const ctx = await seedParents(prisma, "empty"); - await mirrorParents(prismaNew, ctx, "empty"); - - const presenter = new ApiRunListPresenter(prisma, prisma, { - newClient: prismaNew, - legacyReplica: prisma, - splitEnabled: true, - }); - - const result = await presenter.call( - { id: ctx.projectId }, - { "page[size]": 10 } as any, - CURRENT_API_VERSION, - { id: ctx.environmentId, organizationId: ctx.organizationId } - ); - - expect(result.data).toEqual([]); - expect(result.pagination).toHaveProperty("next"); - expect(result.pagination).toHaveProperty("previous"); - } finally { - await prismaNew.$disconnect(); - } + expect(result.data).toEqual([]); + expect(result.pagination).toHaveProperty("next"); + expect(result.pagination).toHaveProperty("previous"); } ); - // Env scoping unchanged: the control-plane runtimeEnvironment.findMany lookup - // resolves the requested env via the `_replica` handle (NOT routed), with the 4th `environment` - // arg omitted to force that branch. Result is scoped to the requested env only. - replicationContainerTest( + containerTest( "env scoping resolves via the control-plane _replica handle (filter[env], 4th arg omitted)", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, - }); - - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; - + async ({ clickhouseContainer, prisma }) => { + const clickhouse = setupClients(prisma, clickhouseContainer.getConnectionUrl()); const ctx = await seedParents(prisma, "scoping", "prod"); - const stagingEnvId = await addEnvironment(prisma, ctx, "scoping", "staging"); + const stagingEnvironmentId = await addEnvironment(prisma, ctx, "scoping", "staging"); - // Runs in prod only; a run in staging must NOT surface when filter[env]=prod. - await createRun(prisma, ctx, { friendlyId: "run_prod1" }); - await createRun(prisma, ctx, { friendlyId: "run_prod2" }); - await createRun(prisma, ctx, { - friendlyId: "run_staging", - runtimeEnvironmentId: stagingEnvId, - }); + // The Postgres rows exercise real hydration; matching ClickHouse rows provide the list IDs. + const runs = await Promise.all([ + createRun(prisma, ctx, { friendlyId: "run_prod1" }), + createRun(prisma, ctx, { friendlyId: "run_prod2" }), + createRun(prisma, ctx, { + friendlyId: "run_staging", + runtimeEnvironmentId: stagingEnvironmentId, + }), + ]); + await insertTaskRunV2Rows(clickhouse, runs); - await setTimeout(1500); - - // Single-handle passthrough; the env lookup runs on `_replica` (= prisma) via findMany. const presenter = new ApiRunListPresenter(prisma, prisma); - // 4th `environment` arg OMITTED -> forces the runtimeEnvironment.findMany branch. + // Omitting the fourth argument forces the control-plane runtimeEnvironment.findMany branch. const result = await presenter.call( { id: ctx.projectId }, { "page[size]": 10, "filter[env]": ["prod"] } as any, CURRENT_API_VERSION ); - // Scoped to the resolved prod env only. - expect(result.data.map((r) => r.id).sort()).toEqual(["run_prod1", "run_prod2"]); + expect(result.data.map((run) => run.id).sort()).toEqual(["run_prod1", "run_prod2"]); } ); - // Passthrough (single-DB): two-arg-style construction (no readThroughDeps) -> - // NextRunListPresenter receives undefined deps -> byte-identical single-DB path. The public - // { data, pagination } shape is unchanged. - replicationContainerTest( + containerTest( "single-DB passthrough: no readThroughDeps lists the seeded runs unchanged", - async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { - const { clickhouse } = await setupClickhouseReplication({ - prisma, - databaseUrl: postgresContainer.getConnectionUri(), - clickhouseUrl: clickhouseContainer.getConnectionUrl(), - redisOptions, - }); - - legacyReplicaHolder.client = prisma; - clickhouseHolder.client = clickhouse; - + async ({ clickhouseContainer, prisma }) => { + const clickhouse = setupClients(prisma, clickhouseContainer.getConnectionUrl()); const ctx = await seedParents(prisma, "passthrough"); - await createRun(prisma, ctx, { friendlyId: "run_pt1" }); - await createRun(prisma, ctx, { friendlyId: "run_pt2" }); + const runs = await Promise.all([ + createRun(prisma, ctx, { friendlyId: "run_pt1" }), + createRun(prisma, ctx, { friendlyId: "run_pt2" }), + ]); + await insertTaskRunV2Rows(clickhouse, runs); - await setTimeout(1500); - - // No readThroughDeps -> passthrough, exactly as the routes construct it today. + // No readThroughDeps preserves the single-database path used by existing callers. const presenter = new ApiRunListPresenter(prisma, prisma); const result = await presenter.call( @@ -402,7 +149,7 @@ describe("ApiRunListPresenter public /runs list (PG14 legacy + PG17 new)", () => { id: ctx.environmentId, organizationId: ctx.organizationId } ); - expect(result.data.map((r) => r.id).sort()).toEqual(["run_pt1", "run_pt2"]); + expect(result.data.map((run) => run.id).sort()).toEqual(["run_pt1", "run_pt2"]); expect(result).toHaveProperty("pagination"); expect(result.pagination).toHaveProperty("next"); expect(result.pagination).toHaveProperty("previous"); diff --git a/apps/webapp/test/dashboardAgentWatches.batch.test.ts b/apps/webapp/test/dashboardAgentWatches.batch.test.ts new file mode 100644 index 000000000..ba71db7bb --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.batch.test.ts @@ -0,0 +1,704 @@ +import { + armWatchBatch, + cancelWatch, + claimWatchBatchTick, + claimWatchDelivery, + createChat, + getWatch, + listActiveWatchesForBatch, + listWatchBatchGroupsToArm, + markWatchDelivered, + recordWatchCheck, + stopWatchBatch, + transitionWatchCondition, + WATCH_DELIVERY_CLAIM_STALE_MS, + type DashboardAgentDb, +} from "@internal/dashboard-agent-db"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import { + DashboardAgentWatchesTestHarness, + HEALTH, + RUN_START, + type DashboardAgentWatchesTestContext, + type Seeded, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => ctx.canAccess, +})); + +const SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.SESSION_SECRET = SESSION_SECRET; + +const { armDashboardAgentWatchBatch, createDashboardAgentWatch, watchBatchStaleMs } = + await import("~/services/dashboardAgentWatches.server"); +const { rearmDashboardAgentWatchBatches } = + await import("~/services/dashboardAgentWatchSweep.server"); +const { runWatchBatchCheck } = await import("~/services/dashboardAgentWatchBatch.server"); +const { signDashboardAgentWatchBatchToken, signDashboardAgentWatchToken } = + await import("~/services/dashboardAgentWatchToken.server"); +const { action: batchCheckAction } = + await import("~/routes/api.v1.dashboard-agent.watches.batch-check"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const authenticated = harness.authenticated.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const fakeCheckDeps = harness.fakeCheckDeps.bind(harness); +const create = harness.create.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +describe("the batch chain registry", () => { + postgresTest("arms one chain per group, and only one", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batcharm"); + const now = new Date(); + + const scheduled: Array<{ epoch: number; tick: number }> = []; + const arm = () => + armDashboardAgentWatchBatch({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + now, + deps: { + schedule: async (params) => + void scheduled.push({ epoch: params.epoch, tick: params.tick }), + }, + }); + + expect(await arm()).toEqual({ running: true }); + expect(scheduled).toEqual([{ epoch: 1, tick: 1 }]); + + expect(await arm()).toEqual({ running: true }); + expect(await arm()).toEqual({ running: true }); + expect(scheduled).toHaveLength(1); + }); + + postgresTest( + "a chain whose run died is re-armed on a fresh epoch, and the zombie claims nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchdead"); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + const scheduled: Array<{ epoch: number; tick: number }> = []; + const arm = (now: Date) => + armDashboardAgentWatchBatch({ + ...group, + now, + deps: { + schedule: async (params) => + void scheduled.push({ epoch: params.epoch, tick: params.tick }), + }, + }); + + const armedAt = new Date(); + await arm(armedAt); + expect( + await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 1 }) + ).toMatchObject({ epoch: 1, generation: 1 }); + + await arm(new Date(armedAt.getTime() + 60_000)); + expect(scheduled).toHaveLength(1); + + await arm(new Date(armedAt.getTime() + watchBatchStaleMs(5) + 60_000)); + expect(scheduled).toEqual([ + { epoch: 1, tick: 1 }, + { epoch: 2, tick: 1 }, + ]); + + expect(await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 2 })).toBe( + null + ); + expect( + await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 2, generation: 1 }) + ).toMatchObject({ epoch: 2, generation: 1 }); + } + ); + + postgresTest( + "a chain that couldn't be triggered is not left marked as running", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchfail"); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + expect( + await armDashboardAgentWatchBatch({ + ...group, + deps: { + schedule: async () => { + throw new Error("the trigger failed"); + }, + }, + }) + ).toEqual({ running: false }); + + const scheduled: number[] = []; + expect( + await armDashboardAgentWatchBatch({ + ...group, + deps: { schedule: async (params) => void scheduled.push(params.epoch) }, + }) + ).toEqual({ running: true }); + expect(scheduled).toEqual([2]); + } + ); + + postgresTest( + "the re-arm backstop finds groups with active watches and no chain", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchrearm"); + await seedChat(seeded); + const created = await create({ + seeded, + spec: HEALTH, + checkDeps: { readHealth: async () => null }, + }); + expect(created.ok).toBe(true); + + const groups = await listWatchBatchGroupsToArm(ctx.agentDb); + expect(groups).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + + const armed: Array<{ environmentId: string; cadenceMinutes: number }> = []; + expect( + await rearmDashboardAgentWatchBatches({ + configured: () => true, + arm: async (params) => { + armed.push({ + environmentId: params.environmentId, + cadenceMinutes: params.cadenceMinutes, + }); + return { running: true }; + }, + }) + ).toEqual({ stale: 1, armed: 1, failed: 0 }); + expect(armed).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + + // The staleness window is the group's own cadence: a five-minute group goes stale 17 minutes later. + await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + staleBefore: new Date(), + }); + expect(await listWatchBatchGroupsToArm(ctx.agentDb)).toEqual([]); + expect( + await listWatchBatchGroupsToArm(ctx.agentDb, { + now: new Date(Date.now() + watchBatchStaleMs(5) + 60_000), + }) + ).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); + } + ); + + postgresTest( + "groups are per environment and per cadence, never mixed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchgroup"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + expect((await create({ seeded, chatId: "chat_1", spec: HEALTH })).ok).toBe(true); + expect((await create({ seeded, chatId: "chat_2", spec: RUN_START })).ok).toBe(true); + + const five = await listActiveWatchesForBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + }); + const one = await listActiveWatchesForBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 1, + }); + + expect(five.map((watch) => watch.chatId)).toEqual(["chat_1"]); + expect(one.map((watch) => watch.chatId)).toEqual(["chat_2"]); + expect( + (await listWatchBatchGroupsToArm(ctx.agentDb)).sort( + (a, b) => a.cadenceMinutes - b.cadenceMinutes + ) + ).toEqual([ + { environmentId: seeded.environment.id, cadenceMinutes: 1 }, + { environmentId: seeded.environment.id, cadenceMinutes: 5 }, + ]); + } + ); +}); + +describe("the batch check", () => { + async function healthGroup(seeded: Seeded, count = 3) { + const ids: string[] = []; + for (let index = 0; index < count; index++) { + const chatId = `chat_${index + 1}`; + await seedChat(seeded, chatId); + const created = await create({ + seeded, + chatId, + spec: HEALTH, + // `warn` keeps them all pending, so the group stays whole for the assertions below. + checkDeps: { readHealth: async () => ({ trustworthy: true, severity: "warn" }) }, + }); + if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); + ids.push(created.watchId); + } + return ids; + } + + async function otherUsersWatch(seeded: Seeded, prisma: PrismaClient) { + const user = await prisma.user.create({ + data: { + email: `other_${seeded.organization.slug}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: user.id, role: "MEMBER" }, + }); + await createChat(ctx.agentDb, { + id: "chat_other", + organizationId: seeded.organization.id, + userId: user.id, + }); + const created = await createDashboardAgentWatch({ + environment: authenticated(seeded), + userId: user.id, + chatId: "chat_other", + spec: HEALTH, + deps: { + configured: () => true, + checkDeps: () => + fakeCheckDeps({ readHealth: async () => ({ trustworthy: true, severity: "warn" }) }), + scheduleTick: async () => {}, + }, + }); + if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); + return { userId: user.id, watchId: created.watchId }; + } + + async function armChain(seeded: Seeded, cadenceMinutes = 5) { + const row = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes, + staleBefore: new Date(), + }); + if (!row) throw new Error("the chain wasn't armed"); + return row; + } + + postgresTest( + "authorizes once and loads the shared report once for the whole group", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchcheck"); + const ids = await healthGroup(seeded); + const chain = await armChain(seeded); + + let healthReads = 0; + let authorizations = 0; + + const response = await runWatchBatchCheck( + { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }, + { + authorize: async () => { + authorizations++; + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => + fakeCheckDeps({ + readHealth: async () => { + healthReads++; + return { trustworthy: true, severity: "warn" }; + }, + }), + } + ); + + expect(authorizations).toBe(1); + expect(healthReads).toBe(1); + + expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual([...ids].sort()); + expect(response.watches?.every((entry) => entry.result === "pending")).toBe(true); + expect(response.watches?.every((entry) => entry.tick === 1)).toBe(true); + expect(response.watches?.every((entry) => entry.token.length > 0)).toBe(true); + expect(response.continues).toBe(true); + expect(response.stale).toBeUndefined(); + + for (const id of ids) { + expect((await getWatch(ctx.agentDb, { id }))?.lastResult).toMatchObject({ + result: "pending", + final: false, + }); + } + } + ); + + postgresTest( + "authorizes each distinct user, so sharing readers never shares access", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchusers"); + await healthGroup(seeded, 2); + const other = await otherUsersWatch(seeded, prisma); + + const chain = await armChain(seeded); + const authorized: string[] = []; + + await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async (watch) => { + authorized.push(watch.userId); + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(authorized.sort()).toEqual([other.userId, seeded.user.id].sort()); + } + ); + + postgresTest( + "cancels a watch whose user lost access, and still answers for its neighbours", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchrevoked"); + const ids = await healthGroup(seeded, 2); + const chain = await armChain(seeded); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async () => ({ ok: false, reason: "access_revoked" }), + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(response.watches?.every((entry) => entry.code === "access_revoked")).toBe(true); + for (const id of ids) { + expect(await getWatch(ctx.agentDb, { id })).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + } + } + ); + + postgresTest( + "checks what is due, skips what isn't, and never skips a window boundary", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchdue"); + const [fresh, overdue, boundary] = await healthGroup(seeded, 3); + const chain = await armChain(seeded); + const now = new Date(); + + await recordWatchCheck(ctx.agentDb, { id: fresh!, lastCheckedAt: now }); + await recordWatchCheck(ctx.agentDb, { + id: overdue!, + lastCheckedAt: new Date(now.getTime() - 10 * 60_000), + }); + // `boundary`'s window closes before the next tick, so its final evaluation must still happen. + await recordWatchCheck(ctx.agentDb, { id: boundary!, lastCheckedAt: now }); + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() + interval '1 minute' where id = $1`, + boundary + ); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + now: () => now, + authorize: async () => ({ ok: true, environment: authenticated(seeded) }), + checkDeps: () => fakeCheckDeps(), + } + ); + + expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual( + [boundary!, overdue!].sort() + ); + expect(response.continues).toBe(true); + } + ); + + postgresTest( + "a stale tick claims nothing and checks nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchstale"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch }; + expect((await runWatchBatchCheck({ ...group, tick: 1 })).stale).toBeUndefined(); + expect((await runWatchBatchCheck({ ...group, tick: 2 })).stale).toBeUndefined(); + + const late = await runWatchBatchCheck({ ...group, tick: 1 }); + expect(late).toEqual({ stale: true }); + + expect(await runWatchBatchCheck({ ...group, epoch: chain.epoch - 1, tick: 1 })).toEqual({ + stale: true, + }); + expect((await getWatch(ctx.agentDb, { id: ids[0]! }))?.status).toBe("active"); + } + ); + + postgresTest( + "stops the chain when the group's last watch is gone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchempty"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + await cancelWatch(ctx.agentDb, { id: ids[0]!, reason: "user" }); + + const response = await runWatchBatchCheck({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }); + + expect(response).toMatchObject({ watches: [], continues: false }); + + const rearmed = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + // Deliberately in the past: only a stopped chain can be re-armed this way. + staleBefore: new Date(Date.now() - 60 * 60_000), + }); + expect(rearmed).toMatchObject({ epoch: chain.epoch + 1, status: "running" }); + } + ); + + postgresTest( + "hands the group's owed wakes back for redelivery", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchowed"); + const ids = await healthGroup(seeded, 2); + const chain = await armChain(seeded); + + await transitionWatchCondition(ctx.agentDb, { + id: ids[0]!, + resolution: "condition_met", + lastResult: { verified: true }, + }); + + const response = await runWatchBatchCheck({ + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain.epoch, + tick: 1, + }); + + const owed = response.watches?.filter((entry) => entry.deliverOnly === true) ?? []; + expect(owed.map((entry) => entry.watchId)).toEqual([ids[0]!]); + expect(owed[0]?.tick).toBe(0); + expect( + response.watches?.filter((entry) => !entry.deliverOnly).map((entry) => entry.watchId) + ).toEqual([ids[1]!]); + } + ); + + postgresTest( + "keeps the chain alive while a wake is still owed, even with nothing left to watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchowedlast"); + const ids = await healthGroup(seeded, 1); + const chain = await armChain(seeded); + const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; + + await transitionWatchCondition(ctx.agentDb, { + id: ids[0]!, + resolution: "condition_met", + lastResult: { verified: true }, + }); + + const first = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 1 }); + expect(first.continues).toBe(true); + expect(first.watches?.map((entry) => entry.deliverOnly)).toEqual([true]); + + const claim = await claimWatchDelivery(ctx.agentDb, { + id: ids[0]!, + staleBefore: new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS), + }); + await markWatchDelivered(ctx.agentDb, { id: ids[0]!, claimId: claim!.claimId }); + + const second = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 2 }); + expect(second).toMatchObject({ watches: [], continues: false }); + expect(await stopWatchBatch(ctx.agentDb, { ...group, epoch: chain.epoch })).toBe(null); + } + ); + + postgresTest( + "one watch that throws mid-evaluation costs only that watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchthrow"); + const mine = await healthGroup(seeded, 2); + const theirs = await otherUsersWatch(seeded, prisma); + const chain = await armChain(seeded); + + const response = await runWatchBatchCheck( + { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, + { + authorize: async (watch) => { + if (watch.userId === theirs.userId) throw new Error("the authorization query failed"); + return { ok: true, environment: authenticated(seeded) }; + }, + checkDeps: () => fakeCheckDeps(), + concurrency: 1, + } + ); + + const byId = new Map(response.watches?.map((entry) => [entry.watchId, entry])); + expect(byId.get(theirs.watchId)).toMatchObject({ result: "unavailable" }); + expect((await getWatch(ctx.agentDb, { id: theirs.watchId }))?.status).toBe("active"); + for (const id of mine) { + expect(byId.get(id)).toMatchObject({ result: "pending" }); + } + } + ); +}); + +describe("the batch check endpoint's authorization", () => { + function batchRequest(body: unknown, token?: string) { + return new Request("https://app.trigger.dev/api/v1/dashboard-agent/watches/batch-check", { + method: "POST", + headers: { + ...(token ? { Authorization: `Bearer ${token}` } : {}), + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + } + + const batchToken = (environmentId: string, cadenceMinutes: number) => + signDashboardAgentWatchBatchToken(SESSION_SECRET, { + environmentId, + cadenceMinutes, + expiresAt: new Date(Date.now() + 60 * 60_000), + }); + + postgresTest("refuses a missing or bad token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const body = { environmentId: "env_1", cadenceMinutes: 5, epoch: 1, tick: 1 }; + + expect( + (await batchCheckAction({ request: batchRequest(body), params: {}, context: {} })).status + ).toBe(401); + const watchToken = await signDashboardAgentWatchToken(SESSION_SECRET, { + watchId: "watch_1", + expiresAt: new Date(Date.now() + 60 * 60_000), + }); + expect( + (await batchCheckAction({ request: batchRequest(body, watchToken), params: {}, context: {} })) + .status + ).toBe(401); + }); + + postgresTest( + "refuses a token minted for another group", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const token = await batchToken("env_1", 5); + + const wrongCadence = await batchCheckAction({ + request: batchRequest( + { environmentId: "env_1", cadenceMinutes: 15, epoch: 1, tick: 1 }, + token + ), + params: {}, + context: {}, + }); + expect(wrongCadence.status).toBe(403); + expect(await wrongCadence.json()).toMatchObject({ code: "group_mismatch" }); + + const wrongEnvironment = await batchCheckAction({ + request: batchRequest( + { environmentId: "env_2", cadenceMinutes: 5, epoch: 1, tick: 1 }, + token + ), + params: {}, + context: {}, + }); + expect(wrongEnvironment.status).toBe(403); + } + ); + + postgresTest( + "answers a group it does own, through the real registry", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "batchroute"); + const chain = await armWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + staleBefore: new Date(), + }); + const token = await batchToken(seeded.environment.id, 5); + + const response = await batchCheckAction({ + request: batchRequest( + { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain!.epoch, + tick: 1, + }, + token + ), + params: {}, + context: {}, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ watches: [], continues: false }); + expect( + await stopWatchBatch(ctx.agentDb, { + environmentId: seeded.environment.id, + cadenceMinutes: 5, + epoch: chain!.epoch, + }) + ).toBe(null); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.delivery.test.ts b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts new file mode 100644 index 000000000..3a129633b --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.delivery.test.ts @@ -0,0 +1,1368 @@ +import { + appendChatMessageOnce, + cancelWatch, + chatExists, + claimWatchDelivery, + claimWatchTick, + countUnreadWatchWakes, + countUserMessages, + getChatMessages, + getWatch, + getWatchSubmission, + listActiveWatchesForChat, + listChatIdsWithUnreadWakes, + listRecentWatchWakes, + markWatchDelivered, + readWatchWakeFeed, + recordWatchCheck, + recordWatchSubmissionOutcome, + releaseWatchDelivery, + transitionWatchCondition, + WATCH_DELIVERY_CLAIM_STALE_MS, + type DashboardAgentDb, + type Watch, +} from "@internal/dashboard-agent-db"; +import type { WatchDraft } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import type { WatchCheckDeps } from "~/services/dashboardAgentWatchChecks"; +import { + DashboardAgentWatchesTestHarness, + RUN_START, + draftFor, + type DashboardAgentWatchesTestContext, + type Seeded, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +process.env.SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; + +const { + cancelDashboardAgentWatch, + createDashboardAgentWatch, + deleteChatWithWatches, + listActiveWatchesForChats, + submitDashboardAgentWatch, +} = await import("~/services/dashboardAgentWatches.server"); +const { sweepDashboardAgentWatches, WATCH_DELIVERY_GRACE_MS, WATCH_EXPIRY_GRACE_MS } = + await import("~/services/dashboardAgentWatchSweep.server"); +const { subscribeUserToWatchAlerts } = await import("~/services/dashboardAgentWatchAlerts.server"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const authenticated = harness.authenticated.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const runRow = harness.runRow.bind(harness); +const fakeCheckDeps = harness.fakeCheckDeps.bind(harness); +const create = harness.create.bind(harness); +const storedMessages = harness.storedMessages.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +describe("the chat cascade and the list view", () => { + postgresTest( + "deleting a chat soft-deletes it and cancels its active watches in one call", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "cascade"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + + const mine = await create({ seeded, chatId: "chat_1" }); + const theirs = await create({ seeded, chatId: "chat_2" }); + expect(mine.ok && theirs.ok).toBe(true); + if (!mine.ok || !theirs.ok) return; + + expect( + await deleteChatWithWatches({ + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toEqual({ + deleted: true, + cancelledWatches: 1, + }); + + expect( + await chatExists(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toBe(false); + expect(await getWatch(ctx.agentDb, { id: mine.watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "chat_deleted", + deliveryStatus: "not_required", + }); + expect(await getWatch(ctx.agentDb, { id: theirs.watchId })).toMatchObject({ + status: "active", + }); + } + ); + + postgresTest( + "a user's own cancel leaves one neutral line in the chat, and only one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "usercancel"); + await seedChat(seeded, "chat_1"); + + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const cancel = () => + cancelDashboardAgentWatch({ + watchId: created.watchId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + + expect(await cancel()).toMatchObject({ + cancelled: true, + messages: [ + { + id: `watch-cancelled:${created.watchId}`, + role: "assistant", + parts: [{ type: "text", text: "Stopped watching run run_1." }], + }, + ], + }); + expect(await getWatch(ctx.agentDb, { id: created.watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "user", + deliveryStatus: "not_required", + }); + expect(await storedMessages(seeded, "chat_1")).toMatchObject([ + { id: `watch-cancelled:${created.watchId}`, role: "assistant" }, + ]); + + // The row is no longer active, so the second cancel writes nothing at all. + expect(await cancel()).toEqual({ cancelled: false, messages: [] }); + expect(await storedMessages(seeded, "chat_1")).toHaveLength(1); + } + ); + + postgresTest( + "a chat delete cancels its watches without a line in the chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "silentcancel"); + await seedChat(seeded, "chat_1"); + + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + + await deleteChatWithWatches({ + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + + const rows = await ctx.prisma.$queryRawUnsafe<{ message_id: string }[]>( + `select message_id from trigger_dashboard_agent.chat_messages where chat_id = 'chat_1'` + ); + expect(rows).toEqual([]); + } + ); + + postgresTest( + "aggregates active watches per chat in one query", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "chips"); + await seedChat(seeded, "chat_1"); + await seedChat(seeded, "chat_2"); + + const a = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_1" } }); + const b = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_2" } }); + const c = await create({ seeded, chatId: "chat_2" }); + expect(a.ok && b.ok && c.ok).toBe(true); + + const byChat = await listActiveWatchesForChats({ + chatIds: ["chat_1", "chat_2", "chat_missing"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + expect(byChat.chat_1).toHaveLength(2); + expect(byChat.chat_2).toHaveLength(1); + expect(byChat.chat_missing).toBeUndefined(); + expect(byChat.chat_2![0]).toMatchObject({ + identity: "run_start:run_1", + status: "active", + kind: "run_start", + note: RUN_START.note, + }); + + if (a.ok) await cancelWatch(ctx.agentDb, { id: a.watchId, reason: "user" }); + if (b.ok) await cancelWatch(ctx.agentDb, { id: b.watchId, reason: "user" }); + expect( + ( + await listActiveWatchesForChats({ + chatIds: ["chat_1"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).chat_1 + ).toBeUndefined(); + } + ); + + postgresTest("returns nothing for an empty chat list", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + expect( + await listActiveWatchesForChats({ chatIds: [], organizationId: "org_x", userId: "user_x" }) + ).toEqual({}); + }); +}); + +describe("unread watch wakes", () => { + postgresTest( + "only signals a wake once its delivery landed", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "unread"); + await seedChat(seeded, "chat_1"); + + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const scope = { organizationId: seeded.organization.id, userId: seeded.user.id }; + const recent = { ...scope, deliveredAfter: new Date(Date.now() - 15 * 60 * 1000) }; + + if (!created.watching) throw new Error("expected a watch"); + await transitionWatchCondition(ctx.agentDb, { + id: created.watchId, + resolution: "condition_met", + }); + expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(0); + expect(await listRecentWatchWakes(ctx.agentDb, recent)).toEqual([]); + expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set()); + + await markWatchDelivered(ctx.agentDb, { id: created.watchId }); + expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(1); + expect(await listRecentWatchWakes(ctx.agentDb, recent)).toMatchObject([ + { watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }, + ]); + expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set(["chat_1"])); + + // The poll's single query answers both halves the same way. + expect(await readWatchWakeFeed(ctx.agentDb, recent)).toMatchObject({ + unreadWakes: 1, + wakes: [{ watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }], + }); + + // An unread wake from before the window still counts, but isn't narrated again. + expect( + await readWatchWakeFeed(ctx.agentDb, { + ...scope, + deliveredAfter: new Date(Date.now() + 60_000), + }) + ).toMatchObject({ unreadWakes: 1, wakes: [] }); + } + ); +}); + +describe("the watch sweep", () => { + async function overdueWatch(seeded: Seeded, chatId = "chat_1") { + const created = await create({ seeded, chatId }); + if (!created.ok) throw new Error("the watch wasn't created"); + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 hour' where id = $1`, + created.watchId + ); + return created.watchId; + } + + function sweepDeps(args: { + seeded: Seeded; + checkDeps?: Partial; + revoked?: boolean; + now?: Date; + failDelivery?: boolean; + delivered: string[]; + }) { + return { + now: () => args.now ?? new Date(), + checkDeps: () => fakeCheckDeps(args.checkDeps), + authorize: async () => + args.revoked + ? ({ ok: false, reason: "access_revoked" } as const) + : ({ ok: true, environment: authenticated(args.seeded) } as const), + deliver: async (watch: Watch) => { + if (args.failDelivery) throw new Error("the delivery couldn't be scheduled"); + args.delivered.push(watch.id); + }, + configured: () => true, + }; + } + + postgresTest( + "runs the final check on an overdue watch and fires it at the buzzer", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches( + sweepDeps({ + seeded, + delivered, + checkDeps: { + readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), + }, + }) + ); + + expect(result).toMatchObject({ overdue: 1, fired: 1, expired: 0, cancelled: 0, failed: 0 }); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "fired", + deliveryStatus: "pending", + }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "expires an overdue watch the check says hasn't happened, as verified", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); + + expect(result).toMatchObject({ overdue: 1, expired: 1, failed: 0 }); + const row = await getWatch(ctx.agentDb, { id: watchId }); + expect(row).toMatchObject({ status: "expired", deliveryStatus: "pending" }); + expect(row?.lastResult).toMatchObject({ verified: true, reason: "not_met_by_expiry" }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "cancels an overdue watch whose user lost access, and never wakes the chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches( + sweepDeps({ seeded, delivered, revoked: true }) + ); + + expect(result).toMatchObject({ overdue: 1, cancelled: 1, expired: 0, fired: 0, failed: 0 }); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "leaves a watch that is still inside its deadline alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + + const delivered: string[] = []; + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); + + expect(result).toMatchObject({ overdue: 0, undelivered: 0 }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "recovers a wake the delivery lost, through the real query, exactly once", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + await expect( + sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, failDelivery: true })) + ).rejects.toThrow(/failed on 1 watches/); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "expired", + deliveryStatus: "pending", + }); + expect(delivered).toEqual([]); + + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const second = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + expect(second).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(delivered).toEqual([watchId]); + + await markWatchDelivered(ctx.agentDb, { id: watchId }); + const third = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + expect(third).toMatchObject({ undelivered: 0, redelivered: 0 }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "a deliverer that died mid-delivery is recovered, but a fresh claim is left alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] })); + + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_status = 'delivering', + delivery_claimed_at = now(), + last_checked_at = now() - interval '1 hour' + where id = $1`, + watchId + ); + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] }))).toMatchObject({ + undelivered: 0, + }); + + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_claimed_at = now() - interval '1 hour' where id = $1`, + watchId + ); + const recovered: string[] = []; + expect( + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: recovered })) + ).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(recovered).toEqual([watchId]); + } + ); + + postgresTest( + "leaves nothing owed for a request the immediate check already answered", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + + const created = await create({ + seeded, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + expect(created.ok).toBe(true); + if (!created.ok || created.watching) throw new Error("expected a one-shot result"); + + const delivered: string[] = []; + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); + + expect(result).toMatchObject({ overdue: 0, undelivered: 0, redelivered: 0 }); + expect(delivered).toEqual([]); + } + ); + + postgresTest( + "finalizes overdue watches even with no agent to deliver to, and delivers once it's back", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const watchId = await overdueWatch(seeded); + + const delivered: string[] = []; + const unconfigured = await sweepDashboardAgentWatches({ + ...sweepDeps({ seeded, delivered }), + configured: () => false, + }); + + expect(unconfigured).toMatchObject({ + overdue: 1, + expired: 1, + deliveryDeferred: 1, + undelivered: 0, + redelivered: 0, + failed: 0, + }); + expect(delivered).toEqual([]); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + status: "expired", + deliveryStatus: "pending", + }); + + const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); + const restored = await sweepDashboardAgentWatches( + sweepDeps({ seeded, delivered, now: later }) + ); + expect(restored).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); + expect(delivered).toEqual([watchId]); + } + ); + + postgresTest( + "the expiry grace keeps the sweep off a watch the tick chain is still finishing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "sweep"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + // A second past the deadline, so the chain's own final check owns this window. + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 second' where id = $1`, + created.watchId + ); + const delivered: string[] = []; + expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered }))).toMatchObject({ + overdue: 0, + }); + + const later = new Date(Date.now() + WATCH_EXPIRY_GRACE_MS + 60_000); + expect( + await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })) + ).toMatchObject({ overdue: 1, expired: 1 }); + } + ); +}); + +describe("the tick claim", () => { + postgresTest( + "claiming a generation is not an observation: only a recorded check stamps one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim"); + await seedChat(seeded); + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) return; + + const claimed = await claimWatchTick(ctx.agentDb, { id: created.watchId, generation: 1 }); + expect(claimed).toMatchObject({ tickCount: 1, lastCheckedAt: null, lastResult: null }); + + await recordWatchCheck(ctx.agentDb, { id: created.watchId, lastResult: { pending: 4 } }); + const row = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(row?.lastCheckedAt).toBeInstanceOf(Date); + expect(row?.lastResult).toMatchObject({ pending: 4 }); + expect(row?.tickCount).toBe(1); + } + ); +}); + +// The delivery claim's fencing token: a hung deliverer is taken over, so an unfenced release or mark would touch the new owner's claim. +describe("the delivery claim", () => { + async function firedWatch(seeded: Seeded) { + const created = await create({ seeded }); + expect(created.ok).toBe(true); + if (!created.ok) throw new Error("the watch wasn't created"); + const transitioned = await transitionWatchCondition(ctx.agentDb, { + id: created.watchId, + status: "fired", + lastResult: { result: "satisfied", facts: { verified: true } }, + }); + expect(transitioned).toMatchObject({ deliveryStatus: "pending" }); + return created.watchId; + } + + function staleBefore() { + return new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS); + } + + async function ageClaim(watchId: string) { + await ctx.prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches + set delivery_claimed_at = now() - interval '1 hour' where id = $1`, + watchId + ); + } + + postgresTest( + "a stale takeover makes the old owner's release a no-op, and the new owner delivers once", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-fence"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(a).not.toBeNull(); + if (!a) return; + + await ageClaim(watchId); + const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(b).not.toBeNull(); + if (!b) return; + expect(b.claimId).not.toBe(a.claimId); + + expect( + await releaseWatchDelivery(ctx.agentDb, { id: watchId, claimId: a.claimId }) + ).toBeNull(); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivering", + deliveryClaimId: b.claimId, + }); + + expect( + await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) + ).toBeNull(); + + expect( + await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) + ).toMatchObject({ deliveryStatus: "delivered" }); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId })).toBeNull(); + } + ); + + postgresTest( + "a late delivered-mark from the old owner completes nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-late"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(a).not.toBeNull(); + if (!a) return; + await ageClaim(watchId); + const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); + expect(b).not.toBeNull(); + if (!b) return; + + expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: a.claimId })).toBeNull(); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); + expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivering", + deliveredAt: null, + }); + + expect( + await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) + ).toMatchObject({ deliveryStatus: "delivered" }); + } + ); + + postgresTest( + "the inline path marks a pending delivery without a claim", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "claim-inline"); + await seedChat(seeded); + const watchId = await firedWatch(seeded); + + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toMatchObject({ + deliveryStatus: "delivered", + }); + expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); + expect( + await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) + ).toBeNull(); + } + ); +}); + +describe("deleting a chat while a watch is being created", () => { + postgresTest("holds in both orders", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + + for (const deleteFirst of [true, false]) { + const chatId = `chat_${deleteFirst ? "del" : "add"}`; + await seedChat(seeded, chatId); + + const creating = () => create({ seeded, chatId }); + const deleting = () => + deleteChatWithWatches({ + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + const [a, b] = deleteFirst + ? await Promise.all([deleting(), creating()]) + : await Promise.all([creating(), deleting()]); + expect(a).toBeDefined(); + expect(b).toBeDefined(); + + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId })).toEqual([]); + expect( + await chatExists(ctx.agentDb, { + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) + ).toBe(false); + } + }); + + postgresTest( + "refuses a create against an already-deleted chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + await seedChat(seeded); + await deleteChatWithWatches({ + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + + expect(await create({ seeded })).toMatchObject({ ok: false, code: "chat_not_found" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); +}); + +describe("appendChatMessageOnce", () => { + postgresTest( + "appends in order without rewriting the transcript", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "append"); + await seedChat(seeded); + + const first = { id: "watch-card:watch_1", role: "assistant", parts: [] }; + const second = { id: "watch-card:watch_2", role: "assistant", parts: [] }; + + expect( + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + message: first, + }) + ).toBe(true); + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + message: second, + }); + + const messages = await getChatMessages(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + expect(messages).toEqual([first, second]); + } + ); + + postgresTest( + "appends nothing for a chat the caller doesn't own", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "append-owner"); + await seedChat(seeded); + + expect( + await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: "user_someone_else", + organizationId: seeded.organization.id, + message: { id: "watch-card:watch_1", role: "assistant", parts: [] }, + }) + ).toBe(false); + + const messages = await getChatMessages(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + }); + expect(messages).toEqual([]); + } + ); +}); + +function submit(args: { + seeded: Seeded; + draft?: WatchDraft; + chatId?: string; + clientRequestId?: string; + checkDeps?: Partial; + subscribed?: boolean; + /** Replaces the fake outright, so a test can hand the submit the real subscribe. */ + subscribe?: typeof subscribeUserToWatchAlerts; + onSchedule?: () => void; + /** Wraps the creation step, so a test can die at the exact point after it. */ + create?: typeof createDashboardAgentWatch; +}) { + return submitDashboardAgentWatch({ + environment: authenticated(args.seeded), + userId: args.seeded.user.id, + organizationId: args.seeded.organization.id, + chatId: args.chatId, + clientRequestId: args.clientRequestId ?? "wreq_1", + draft: args.draft ?? draftFor(RUN_START), + deps: { + configured: () => true, + checkDeps: () => fakeCheckDeps(args.checkDeps), + scheduleTick: async () => args.onSchedule?.(), + ...(args.create ? { create: args.create } : {}), + subscribe: + args.subscribe ?? + (async () => + args.subscribed === false + ? { ok: false, reason: "dashboard_agent_disabled" } + : { ok: true, email: args.seeded.user.email }), + }, + }); +} + +describe("the watch card submit", () => { + postgresTest( + "records what the user confirmed before the watch, and confirms it after", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(true); + expect(result.repaired).toBe(false); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${result.watchId}`, + ]); + // The consent record is the user's, and it states the condition and the lifetime. + expect(stored?.[0]).toMatchObject({ role: "user" }); + expect(JSON.stringify(stored?.[0])).toContain("Watch run run_1 until it starts."); + expect(JSON.stringify(stored?.[0])).toContain("Investigate straight away"); + expect(result.messages.map((message) => message.id)).toEqual( + stored?.map((message) => message.id) + ); + } + ); + + postgresTest( + "leaves a repairable state when the confirmation never lands, and the retry repairs it", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-repair"); + await seedChat(seeded); + + // The crash state: the request record is written and the watch is live, but the + // process died before the confirmation was appended. + const requestAppended = await appendChatMessageOnce(ctx.agentDb, { + chatId: "chat_1", + userId: seeded.user.id, + organizationId: seeded.organization.id, + message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, + }); + expect(requestAppended).toBe(true); + const created = await create({ seeded, chatId: "chat_1" }); + expect(created.ok).toBe(true); + if (!created.ok || !created.watching) return; + + const retry = await submit({ seeded, chatId: "chat_1", clientRequestId: "wreq_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(created.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${created.watchId}`, + ]); + + // Still exactly one watch: the repair loaded it rather than creating another. + const active = await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" }); + expect(active).toHaveLength(1); + } + ); + + postgresTest( + "a retried submit duplicates neither record", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-retry"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + const second = await submit({ seeded, chatId: "chat_1" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.repaired).toBe(true); + expect(second.watchId).toBe(first.watchId); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a genuinely different request still conflicts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-conflict"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + // Same condition, so the same identity, but a different window: not a retry. + const longer = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_2", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(longer).toMatchObject({ ok: false, code: "duplicate", existingId: first.watchId }); + + // Same spec, different consent: also not a retry. + const investigating = await submit({ + seeded, + chatId: "chat_1", + clientRequestId: "wreq_3", + draft: draftFor(RUN_START, { investigateOnAttention: true }), + }); + expect(investigating).toMatchObject({ ok: false, code: "duplicate" }); + + // The refused attempts are recorded under their own consent records, so the + // transcript never shows a request with no answer. + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + "watch-request:wreq_2", + "watch-confirmation:refused:wreq_2", + "watch-request:wreq_3", + "watch-confirmation:refused:wreq_3", + ]); + } + ); + + postgresTest( + "a fresh panel's retry reuses the chat the first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fresh"); + + const first = await submit({ seeded, clientRequestId: "wreq_fresh" }); + const second = await submit({ seeded, clientRequestId: "wreq_fresh" }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) return; + expect(second.chatId).toBe(first.chatId); + + const stored = await storedMessages(seeded, first.chatId); + expect(stored).toHaveLength(2); + } + ); + + postgresTest( + "an answered condition records the request and a one-shot result, and never a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot"); + await seedChat(seeded); + + const result = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + expect(result.watchId).toBeNull(); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + /** Every watch row for a chat, terminal ones included. `listActiveWatchesForChat` can't see those. */ + async function countWatchRows(prisma: PrismaClient, chatId: string) { + const rows = await prisma.$queryRawUnsafe>( + `select count(*)::bigint as count from trigger_dashboard_agent.watches where chat_id = $1`, + chatId + ); + return Number(rows[0]?.count ?? 0); + } + + postgresTest( + "a retry after the watch has already fired creates no second watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-fired"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok || !first.watchId) return; + + // The watch resolves and leaves the active set, so a duplicate check would find + // nothing. Only the ledger still knows this request already ran. + await transitionWatchCondition(ctx.agentDb, { + id: first.watchId, + resolution: "condition_met", + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a retry of an answered one-shot never becomes a watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-oneshot-retry"); + await seedChat(seeded); + + const first = await submit({ + seeded, + chatId: "chat_1", + checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, + }); + expect(first.ok && first.watching === false).toBe(true); + + // The world moved on: the same condition would now be pending, so a re-evaluation + // would start a real watch. The recorded outcome is replayed instead. + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.watching).toBe(false); + expect(retry.watchId).toBeNull(); + expect(retry.repaired).toBe(true); + expect(await countWatchRows(prisma, "chat_1")).toBe(0); + + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + "watch-confirmation:one-shot:wreq_1", + ]); + } + ); + + postgresTest( + "the same request id carrying a different draft is a conflict", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-hash"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok) return; + + const changed = await submit({ + seeded, + chatId: "chat_1", + draft: draftFor({ ...RUN_START, maxHours: 6 }), + }); + expect(changed).toMatchObject({ ok: false, code: "request_conflict" }); + + // A conflict writes nothing at all: no watch, and no record under the request. + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + const stored = await storedMessages(seeded, "chat_1"); + expect(stored?.map((message) => message.id)).toEqual([ + "watch-request:wreq_1", + `watch-confirmation:${first.watchId}`, + ]); + } + ); + + postgresTest( + "a pending submission converges on the watch its first attempt created", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-converge"); + await seedChat(seeded); + + // The crash state the ledger exists for: the row is reserved, the watch is live + // under the reserved id, and the process died before the outcome was written. + let reservedWatchId = ""; + await expect( + submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + await createDashboardAgentWatch(createParams); + throw new Error("died after the watch was created"); + }, + }) + ).rejects.toThrow("died after the watch was created"); + + const pending = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(pending).toMatchObject({ state: "pending", watchId: reservedWatchId }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + // Reached the reserved row rather than creating another. + expect(retry.watchId).toBe(reservedWatchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const settled = await getWatchSubmission(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + }); + expect(settled).toMatchObject({ state: "created", watchId: reservedWatchId }); + } + ); + + postgresTest( + "converging on a watch that already fired confirms the outcome, not 'watching'", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-converge-fired"); + await seedChat(seeded); + + let reservedWatchId = ""; + await expect( + submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + await createDashboardAgentWatch(createParams); + throw new Error("died after the watch was created"); + }, + }) + ).rejects.toThrow("died after the watch was created"); + + // The watch ran and woke the chat before anyone retried the submit. + await transitionWatchCondition(ctx.agentDb, { + id: reservedWatchId, + resolution: "condition_met", + observedOutcome: { kind: "run_start", verified: true, status: "EXECUTING", started: true }, + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + // Still one row, still the same watch: adoption is not refused. + expect(retry.watchId).toBe(reservedWatchId); + expect(await countWatchRows(prisma, "chat_1")).toBe(1); + + const parts = retry.messages.at(-1)?.parts ?? []; + const block = (parts[0] as any).data.blocks[0]; + expect(block.outcome).toBe("already_true"); + expect(block.headline).not.toContain("Watching"); + expect(block.lifetime).toBeNull(); + } + ); + + postgresTest( + "a refusal that wins the race leaves no live watch behind", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-refused-race"); + await seedChat(seeded); + + // A concurrent attempt refuses this submission after the watch exists under the + // reserved id, so the ledger's winner keeps naming that id. + let reservedWatchId = ""; + const result = await submit({ + seeded, + chatId: "chat_1", + create: async (createParams) => { + reservedWatchId = createParams.watchId!; + const created = await createDashboardAgentWatch(createParams); + const refused = await recordWatchSubmissionOutcome(ctx.agentDb, { + chatId: "chat_1", + clientRequestId: "wreq_1", + state: "refused", + refusalCode: "internal", + refusalError: "That watch couldn't be started.", + }); + expect(refused).toMatchObject({ state: "refused", watchId: reservedWatchId }); + return created; + }, + }); + + // The user is told nothing is being watched, so nothing may be watching. + expect(result.ok).toBe(false); + const row = await getWatch(ctx.agentDb, { id: reservedWatchId }); + expect(row).toMatchObject({ status: "cancelled", cancelReason: "superseded" }); + } + ); + + postgresTest( + "the consent record never spends a message from the cap", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-quota"); + await seedChat(seeded); + + await submit({ seeded, chatId: "chat_1" }); + + expect( + await countUserMessages(ctx.agentDb, { + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).toBe(0); + } + ); + + postgresTest( + "a replay repeats the recorded email outcome and subscribes nobody", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-external-replay"); + await seedChat(seeded); + + const draft = draftFor(RUN_START, { notifyExternally: true }); + + // The first attempt asked for email and couldn't get it, so `unavailable` is what + // the transcript says and what the ledger records. + const first = await submit({ seeded, chatId: "chat_1", draft, subscribed: false }); + expect(first.ok).toBe(true); + if (!first.ok) return; + expect(JSON.stringify(first.messages)).toContain("I couldn't add email notifications"); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ state: "created", externalNotificationStatus: "unavailable" }); + + const transcript = await storedMessages(seeded, "chat_1"); + + // The retry gets the real subscribe, which would succeed here. A replay that took the + // decision again would leave a channel row and an `enabled` answer the transcript โ€” + // append-once, so never rewritten โ€” contradicts for good. + let subscribeCalls = 0; + const retry = await submit({ + seeded, + chatId: "chat_1", + draft, + subscribe: async (subscribeParams) => { + subscribeCalls++; + return subscribeUserToWatchAlerts(subscribeParams); + }, + }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + expect(retry.watchId).toBe(first.watchId); + expect(subscribeCalls).toBe(0); + + expect(JSON.stringify(retry.messages)).toContain("I couldn't add email notifications"); + expect(JSON.stringify(retry.messages)).not.toContain("You'll get an email"); + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(0); + expect( + await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) + ).toMatchObject({ externalNotificationStatus: "unavailable" }); + + // The symptom: what the user is told after a refresh has to agree with the answer. + expect(await storedMessages(seeded, "chat_1")).toEqual(transcript); + } + ); + + postgresTest( + "a replay repeats the recorded 'Watching' confirmation after the watch has fired", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "submit-replay-fired"); + await seedChat(seeded); + + const first = await submit({ seeded, chatId: "chat_1" }); + expect(first.ok).toBe(true); + if (!first.ok || !first.watchId) return; + + await transitionWatchCondition(ctx.agentDb, { + id: first.watchId, + resolution: "condition_met", + }); + + const retry = await submit({ seeded, chatId: "chat_1" }); + + expect(retry.ok).toBe(true); + if (!retry.ok) return; + expect(retry.repaired).toBe(true); + + // The recorded outcome is replayed, never decided again: the append-once + // confirmation in the transcript says "Watching", so the answer has to as well. + const parts = retry.messages.at(-1)?.parts ?? []; + const block = (parts[0] as any).data.blocks[0]; + expect(block.outcome).toBe("watching"); + expect(block.headline).toContain("Watching"); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts new file mode 100644 index 000000000..8029cae9f --- /dev/null +++ b/apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts @@ -0,0 +1,1302 @@ +import { + createChat, + getWatch, + listActiveWatchesForChat, + recordWatchCheck, + type DashboardAgentDb, +} from "@internal/dashboard-agent-db"; +import type { WatchSpec } from "@internal/dashboard-agent-contracts"; +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, beforeEach, describe, expect, vi } from "vitest"; +import { previousCheckFacts } from "~/services/dashboardAgentWatchChecks"; +import { + BACKLOG, + DashboardAgentWatchesTestHarness, + RUN_START, + readRunOnce, + type DashboardAgentWatchesTestContext, + type Seeded, +} from "./helpers/dashboardAgentWatchesTestHelpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const ctx = vi.hoisted( + (): DashboardAgentWatchesTestContext => ({ + prisma: undefined as unknown as PrismaClient, + agentDb: undefined as unknown as DashboardAgentDb, + canAccess: true, + actor: undefined, + triggered: [], + }) +); + +vi.mock("~/services/uatRoutePreamble.server", () => ({ + authenticateUatOrApiRequest: async () => + ctx.actor + ? { + authenticationResult: { + type: "personalAccessToken", + result: { userId: ctx.actor.userId }, + }, + userActor: ctx.actor, + } + : undefined, +})); + +vi.mock("~/db.server", () => { + const proxy = new Proxy( + {}, + { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } + ); + return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; +}); + +vi.mock("~/services/dashboardAgentDb.server", () => ({ + get dashboardAgentDb() { + return ctx.agentDb; + }, +})); + +vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ + canAccessDashboardAgent: async () => ctx.canAccess, +})); + +vi.mock("@trigger.dev/sdk", async (importOriginal) => { + const actual = await importOriginal>(); + return { + ...actual, + TriggerClient: class { + tasks = { + trigger: async (taskId: string) => { + ctx.triggered.push(taskId); + return { id: "run_test" }; + }, + }; + }, + }; +}); + +const SESSION_SECRET = "test-session-secret-for-watch-tokens"; +process.env.SESSION_SECRET = SESSION_SECRET; +process.env.ALERT_FROM_EMAIL = "alerts@example.com"; +process.env.ALERT_EMAIL_TRANSPORT = "smtp"; +process.env.DASHBOARD_AGENT_SECRET_KEY = "test-dashboard-agent-secret"; + +const { authorizeWatchEnvironment, createDashboardAgentWatch, listActiveWatchesForChats } = + await import("~/services/dashboardAgentWatches.server"); +const { action: checkAction } = + await import("~/routes/api.v1.dashboard-agent.watches.$watchId.check"); +const { action: createAction } = await import("~/routes/api.v1.dashboard-agent.watches"); +const { signDashboardAgentWatchToken } = await import("~/services/dashboardAgentWatchToken.server"); +const { loader: alertsLoader, action: alertsAction } = + await import("~/routes/api.v1.dashboard-agent.alerts"); +const { action: alertChannelAction } = + await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); +const { findProjectBySlug } = await import("~/models/project.server"); +const { DASHBOARD_AGENT_WATCH_ALERT_TYPE } = + await import("~/services/dashboardAgentWatchAlerts.server"); + +const harness = new DashboardAgentWatchesTestHarness(ctx, createDashboardAgentWatch); +const boot = harness.boot.bind(harness); +const seed = harness.seed.bind(harness); +const seedChat = harness.seedChat.bind(harness); +const runRow = harness.runRow.bind(harness); +const create = harness.create.bind(harness); + +beforeEach(() => harness.reset()); +afterEach(() => harness.close()); + +describe("createDashboardAgentWatch", () => { + postgresTest( + "creates an active watch and schedules its first tick", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const scheduled: Array<{ watchId: string; token: string; tick: number }> = []; + const result = await create({ seeded, scheduled }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.status).toBe("active"); + expect(result.identity).toBe("run_start:run_1"); + expect(result.immediate).toBeUndefined(); + + expect(scheduled).toHaveLength(1); + expect(scheduled[0]!.watchId).toBe(result.watchId); + expect(scheduled[0]!.tick).toBe(1); + expect(scheduled[0]!.token.startsWith("tr_daw_")).toBe(true); + + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + expect(row).toMatchObject({ + status: "active", + deliveryStatus: "not_required", + environmentId: seeded.environment.id, + projectId: seeded.project.id, + organizationId: seeded.organization.id, + userId: seeded.user.id, + tickCount: 0, + investigateOnAttention: false, + projectRef: seeded.project.externalRef, + }); + } + ); + + postgresTest( + "records the investigate-on-attention consent when the caller asks for it", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ seeded, investigateOnAttention: true }); + + expect(result.ok).toBe(true); + if (!result.ok || !result.watching) return; + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + expect(row?.investigateOnAttention).toBe(true); + expect(result.identity).toBe("run_start:run_1"); + } + ); + + postgresTest( + "stamps a server-set `since` on an error_recurrence watch", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const before = Date.now(); + const result = await create({ + seeded, + spec: { + kind: "error_recurrence", + fingerprint: "fp_1", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it comes back", + }, + }); + expect(result.ok).toBe(true); + if (!result.ok) return; + + const row = await getWatch(ctx.agentDb, { id: result.watchId }); + const since = (row?.spec as { since?: string } | undefined)?.since; + expect(since).toBeDefined(); + expect(new Date(since!).getTime()).toBeGreaterThanOrEqual(before - 1000); + } + ); + + postgresTest( + "answers with a one-shot result and writes no row when the condition already holds", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + let ticks = 0; + const result = await create({ + seeded, + checkDeps: { + readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), + }, + onSchedule: () => { + ticks += 1; + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok || result.watching) throw new Error("expected a one-shot result"); + expect(result.immediate.result).toBe("satisfied"); + expect(result.immediate.observed).toMatchObject({ kind: "run_start", started: true }); + expect(ticks).toBe(0); + + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + expect( + await listActiveWatchesForChats({ + chatIds: ["chat_1"], + organizationId: seeded.organization.id, + userId: seeded.user.id, + }) + ).toEqual({}); + } + ); + + postgresTest( + "answers with a one-shot result when the condition can no longer happen", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + checkDeps: { readRun: readRunOnce(runRow({ status: "QUEUED" })) }, + }); + + expect(result.ok).toBe(true); + if (!result.ok || result.watching) throw new Error("expected a one-shot result"); + expect(result.immediate.result).toBe("terminal_unsatisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses a duplicate before running the immediate check", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const first = await create({ seeded }); + expect(first.ok).toBe(true); + + let checks = 0; + const second = await create({ + seeded, + checkDeps: { + readRun: async () => { + checks += 1; + return runRow({ status: "EXECUTING", startedAt: new Date() }); + }, + }, + }); + + expect(second).toMatchObject({ ok: false, code: "duplicate" }); + expect(checks).toBe(1); + } + ); + + postgresTest( + "cancels the row silently when the first tick can't be scheduled", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + onSchedule: () => { + throw new Error("no agent project"); + }, + }); + + expect(result).toMatchObject({ ok: false, code: "internal" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + const rows = await ctx.prisma.$queryRawUnsafe< + { status: string; cancel_reason: string; delivery_status: string }[] + >( + `select status, cancel_reason, delivery_status + from trigger_dashboard_agent.watches where chat_id = 'chat_1'` + ); + expect(rows).toMatchObject([ + { + status: "cancelled", + cancel_reason: "scheduling_failed", + delivery_status: "not_required", + }, + ]); + } + ); + + postgresTest( + "rejects a target that doesn't exist, writing nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: BACKLOG, + checkDeps: { queueExists: async () => false }, + }); + + expect(result).toMatchObject({ ok: false, code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "dedups the same condition and allows it in another environment", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + const first = await create({ seeded }); + expect(first.ok).toBe(true); + + const second = await create({ seeded }); + expect(second).toMatchObject({ ok: false, code: "duplicate" }); + if (!second.ok && first.ok) expect(second.existingId).toBe(first.watchId); + + const otherEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "stg", + type: "STAGING", + projectId: seeded.project.id, + organizationId: seeded.organization.id, + apiKey: `tr_stg_${seeded.project.slug}`, + pkApiKey: `pk_stg_${seeded.project.slug}`, + shortcode: `s${seeded.project.slug.slice(0, 6)}`, + }, + }); + const third = await create({ seeded, environmentId: otherEnv.id }); + expect(third.ok).toBe(true); + } + ); + + postgresTest( + "refuses a 4th active watch in the same chat", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "watch"); + await seedChat(seeded); + + for (const runId of ["run_1", "run_2", "run_3"]) { + const created = await create({ seeded, spec: { ...RUN_START, runId } }); + expect(created.ok).toBe(true); + } + + const fourth = await create({ seeded, spec: { ...RUN_START, runId: "run_4" } }); + expect(fourth).toMatchObject({ ok: false, code: "limit_reached" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); + } + ); + + postgresTest( + "holds the โ‰ค3 limit against four concurrent creates", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "race"); + await seedChat(seeded); + + const results = await Promise.all( + ["run_1", "run_2", "run_3", "run_4"].map((runId) => + create({ seeded, spec: { ...RUN_START, runId } }) + ) + ); + + expect(results.filter((result) => result.ok)).toHaveLength(3); + expect( + results.filter((result) => !result.ok && result.code === "limit_reached") + ).toHaveLength(1); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); + } + ); +}); + +describe("authorizeWatchEnvironment", () => { + postgresTest( + "passes for a member and fails once membership is gone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + + const params = { + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: seeded.environment.id, + }; + + expect((await authorizeWatchEnvironment(params)).ok).toBe(true); + + await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); + expect(await authorizeWatchEnvironment(params)).toEqual({ + ok: false, + reason: "access_revoked", + }); + } + ); + + postgresTest("fails when the feature gate is revoked", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + ctx.canAccess = false; + + expect( + await authorizeWatchEnvironment({ + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + environmentId: seeded.environment.id, + }) + ).toEqual({ ok: false, reason: "access_revoked" }); + }); + + postgresTest( + "fails when the snapshot names a different project", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "auth"); + const other = await seed(prisma, "other"); + + expect( + await authorizeWatchEnvironment({ + userId: seeded.user.id, + organizationId: seeded.organization.id, + projectId: other.project.id, + environmentId: seeded.environment.id, + }) + ).toEqual({ ok: false, reason: "access_revoked" }); + } + ); +}); + +describe("run_failed creation", () => { + const RUN_FAILED: WatchSpec = { + kind: "run_failed", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me if it fails", + }; + + postgresTest( + "watches a running run and dedups against the finished variant separately", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "runfailed"); + await seedChat(seeded); + + const failed = await create({ + seeded, + spec: RUN_FAILED, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, + }); + expect(failed.ok).toBe(true); + if (!failed.ok || !failed.watching) return; + expect(failed.identity).toBe("run_failed:run_1"); + + const finished = await create({ + seeded, + spec: { ...RUN_FAILED, kind: "run_finished" } as WatchSpec, + checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, + }); + expect(finished.ok).toBe(true); + if (!finished.ok || !finished.watching) return; + expect(finished.identity).toBe("run_finished:run_1"); + } + ); + + postgresTest( + "answers outright, with no watch row, once the run has succeeded", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "runfailed-done"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: RUN_FAILED, + checkDeps: { + readRun: async () => + runRow({ status: "COMPLETED_SUCCESSFULLY", completedAt: new Date() }), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + if (result.watching) return; + expect(result.immediate.result).toBe("terminal_unsatisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); +}); + +describe("the queue pack creation", () => { + const QUEUE = "task/my-task"; + + const BELOW: WatchSpec = { + kind: "queue_depth_below", + queue: QUEUE, + threshold: 100, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when it's back below 100", + }; + + const STALLED: WatchSpec = { + kind: "queue_stalled", + queue: QUEUE, + ticks: 3, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if it stops moving", + }; + + const AGE: WatchSpec = { + kind: "queue_oldest_age", + queue: QUEUE, + thresholdMinutes: 5, + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me if runs wait longer than 5 minutes", + }; + + postgresTest( + "creates each kind with its own identity on the same queue", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuepack"); + await seedChat(seeded); + + const busy = { + readQueueDepth: async () => ({ + depth: 780, + source: "live_queue" as const, + current: true, + }), + }; + + const below = await create({ seeded, spec: BELOW, checkDeps: busy }); + expect(below.ok && below.watching).toBe(true); + if (!below.ok || !below.watching) return; + expect(below.identity).toBe(`queue_depth_below:${QUEUE}:100`); + + const stalled = await create({ seeded, spec: STALLED, checkDeps: busy }); + expect(stalled.ok && stalled.watching).toBe(true); + if (!stalled.ok || !stalled.watching) return; + expect(stalled.identity).toBe(`queue_stalled:${QUEUE}`); + + const age = await create({ seeded, spec: AGE, checkDeps: busy }); + expect(age.ok && age.watching).toBe(true); + if (!age.ok || !age.watching) return; + expect(age.identity).toBe(`queue_oldest_age:${QUEUE}:5`); + + const drain = await create({ + seeded, + spec: { ...BELOW, kind: "backlog_drain" } as WatchSpec, + checkDeps: busy, + }); + expect(drain.ok).toBe(false); + if (drain.ok) return; + expect(drain.code).toBe("limit_reached"); + } + ); + + postgresTest( + "dedups the same SLA and allows a different one", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queueage"); + await seedChat(seeded); + + const first = await create({ seeded, spec: AGE }); + expect(first.ok && first.watching).toBe(true); + + const same = await create({ seeded, spec: AGE }); + expect(same.ok).toBe(false); + if (same.ok) return; + expect(same.code).toBe("duplicate"); + + const other = await create({ seeded, spec: { ...AGE, thresholdMinutes: 30 } as WatchSpec }); + expect(other.ok && other.watching).toBe(true); + if (!other.ok || !other.watching) return; + expect(other.identity).toBe(`queue_oldest_age:${QUEUE}:30`); + } + ); + + postgresTest( + "answers a back-below ask outright when the queue is already quiet", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuebelow"); + await seedChat(seeded); + + const result = await create({ + seeded, + spec: BELOW, + checkDeps: { + readQueueDepth: async () => ({ depth: 4, source: "live_queue", current: true }), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.watching).toBe(false); + if (result.watching) return; + expect(result.immediate.result).toBe("satisfied"); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); + } + ); + + postgresTest( + "round-trips the stall state through the row's existing facts column", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "queuestall"); + await seedChat(seeded); + + const created = await create({ + seeded, + spec: STALLED, + checkDeps: { + readQueueDepth: async () => ({ depth: 42, source: "live_queue", current: true }), + }, + }); + expect(created.ok && created.watching).toBe(true); + if (!created.ok || !created.watching) return; + + const facts = { queue: QUEUE, depth: 42, notDecreasingStreak: 2, ticks: 3 }; + await recordWatchCheck(ctx.agentDb, { + id: created.watchId, + lastResult: { + result: "pending", + facts, + observed: { + kind: "queue_stalled", + verified: true, + depth: 42, + notDecreasingStreak: 2, + ticks: 3, + }, + final: false, + }, + }); + + const row = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(previousCheckFacts(row?.lastResult)).toEqual(facts); + + await recordWatchCheck(ctx.agentDb, { + id: created.watchId, + lastResult: { checkFailed: true, detail: "clickhouse down", previous: facts }, + }); + const afterGap = await getWatch(ctx.agentDb, { id: created.watchId }); + expect(previousCheckFacts(afterGap?.lastResult)).toEqual(facts); + } + ); +}); + +describe("the createWatch endpoint's authorization", () => { + function post(body: unknown) { + return createAction({ + request: new Request("https://example.com/api/v1/dashboard-agent/watches", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {}, + }); + } + + const validBody = (chatId: string) => ({ spec: RUN_START, chatId }); + + postgresTest("401s without a delegated token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const response = await post(validBody("chat_1")); + expect(response.status).toBe(401); + }); + + postgresTest("403s for any other client's token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "adapter"); + ctx.actor = { userId: seeded.user.id, client: "cli", environmentId: seeded.environment.id }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "forbidden_client" }); + }); + + postgresTest( + "refuses a chat the authenticated user doesn't own, writing nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const owner = await seed(prisma, "owner"); + const stranger = await seed(prisma, "stranger"); + await createChat(ctx.agentDb, { + id: "chat_victim", + organizationId: owner.organization.id, + userId: owner.user.id, + }); + + ctx.actor = { + userId: stranger.user.id, + client: "dashboard-agent", + environmentId: stranger.environment.id, + }; + + const response = await post(validBody("chat_victim")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "chat_not_found" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_victim" })).toHaveLength( + 0 + ); + } + ); + + postgresTest( + "refuses a token with no environment scope", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "noscope"); + await seedChat(seeded, "chat_1"); + ctx.actor = { userId: seeded.user.id, client: "dashboard-agent" }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses a body naming a different environment than the token's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "mismatch"); + const other = await seed(prisma, "othermismatch"); + await seedChat(seeded, "chat_1"); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + environmentId: other.environment.id, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "binds to the token's environment, not the chat's stored context", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "binding"); + const otherProject = await prisma.project.create({ + data: { + name: `${seeded.project.slug}_b`, + slug: `${seeded.project.slug}_b`, + organizationId: seeded.organization.id, + externalRef: `proj_${seeded.project.slug}_b`, + }, + }); + const otherEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: otherProject.id, + organizationId: seeded.organization.id, + apiKey: `tr_prod_${otherProject.slug}`, + pkApiKey: `pk_prod_${otherProject.slug}`, + shortcode: `b${otherProject.slug.slice(0, 6)}`, + }, + }); + + await createChat(ctx.agentDb, { + id: "chat_1", + organizationId: seeded.organization.id, + userId: seeded.user.id, + metadata: { + context: { + environmentId: seeded.environment.id, + projectRef: seeded.project.externalRef, + }, + }, + }); + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: otherEnvironment.id, + }; + + const response = await post({ + ...validBody("chat_1"), + projectRef: seeded.project.externalRef, + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); + + postgresTest( + "refuses an environment in another org than the chat's", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "crossorg"); + const other = await seed(prisma, "otherorg"); + await prisma.orgMember.create({ + data: { + organizationId: other.organization.id, + userId: seeded.user.id, + role: "ADMIN", + }, + }); + await seedChat(seeded, "chat_1"); + + ctx.actor = { + userId: seeded.user.id, + client: "dashboard-agent", + environmentId: other.environment.id, + }; + + const response = await post(validBody("chat_1")); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ code: "invalid_target" }); + expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); + } + ); +}); + +describe("the check endpoint", () => { + function request(token: string, body: unknown = {}) { + return new Request("https://example.com/api/v1/dashboard-agent/watches/x/check", { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + } + + async function activeWatch(seeded: Seeded, spec?: WatchSpec) { + const result = await create({ seeded, spec }); + if (!result.ok) throw new Error(`watch not created: ${result.code}`); + if (!result.watching) throw new Error("expected an active watch"); + return result; + } + + function tokenFor(watchId: string, expiresAt: Date) { + return signDashboardAgentWatchToken(SESSION_SECRET, { watchId, expiresAt }); + } + + postgresTest("401s on a bad token", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + const response = await checkAction({ + request: request("tr_daw_nonsense"), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(401); + }); + + postgresTest("403s when the token names another watch", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor("watch_someone_else", watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "watch_mismatch" }); + }); + + postgresTest("answers a check and records what it saw", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.result).toBe("terminal_unsatisfied"); + + // Arming the chain goes through the stubbed client, never a real trigger. + expect(ctx.triggered).toContain("dashboard-agent-watch-batch"); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row?.lastCheckedAt).not.toBeNull(); + expect(row?.tickCount).toBe(0); + expect(row?.status).toBe("active"); + }); + + postgresTest( + "refuses an ordinary check after expiry but allows the final one in grace", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 minute' where id = $1`, + watch.watchId + ); + + const token = await tokenFor(watch.watchId, watch.expiresAt); + + const refused = await checkAction({ + request: request(token, {}), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(refused.status).toBe(403); + expect(await refused.json()).toMatchObject({ code: "expired" }); + + const allowed = await checkAction({ + request: request(token, { final: true }), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(allowed.status).toBe(200); + } + ); + + postgresTest( + "cancels the watch on revoked access, without reading environment data", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "access_revoked" }); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + expect(row).toMatchObject({ + status: "cancelled", + cancelReason: "access_revoked", + deliveryStatus: "not_required", + }); + expect(row?.tickCount).toBe(0); + expect(row?.lastResult).toBeNull(); + } + ); + + postgresTest( + "a check that couldn't read anything leaves the row's last look and facts alone", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + + // The queue exists, so the check gets past the target read and fails on the depth + // read: there is no live queue or analytics store behind this environment. + const queue = "task/stalling"; + await prisma.taskQueue.create({ + data: { + runtimeEnvironmentId: seeded.environment.id, + projectId: seeded.project.id, + name: queue, + friendlyId: `queue_${Math.random().toString(36).slice(2, 10)}`, + orderableName: queue, + }, + }); + + const watch = await activeWatch(seeded, { + kind: "queue_stalled", + queue, + ticks: 3, + checkEveryMinutes: 5, + maxHours: 6, + note: "tell me if the queue stops moving", + }); + + // Two no-progress checks already behind it, last looked at an hour ago. + const checkedAt = new Date(Date.now() - 60 * 60 * 1000); + await recordWatchCheck(ctx.agentDb, { + id: watch.watchId, + lastCheckedAt: checkedAt, + lastResult: { + result: "pending", + facts: { queue, depth: 412, notDecreasingStreak: 2, ticks: 3 }, + }, + }); + + const token = await tokenFor(watch.watchId, watch.expiresAt); + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ result: "unavailable" }); + + const row = await getWatch(ctx.agentDb, { id: watch.watchId }); + // Nothing was checked, so the watch is still due at the next tick. + expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); + // And the streak the earlier ticks built is still there to be continued. + expect(previousCheckFacts(row?.lastResult)).toMatchObject({ + depth: 412, + notDecreasingStreak: 2, + }); + }, + 120_000 + ); + + postgresTest("403s once the watch is terminal", async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "check"); + await seedChat(seeded); + const watch = await activeWatch(seeded); + const token = await tokenFor(watch.watchId, watch.expiresAt); + + await prisma.$executeRawUnsafe( + `update trigger_dashboard_agent.watches set status = 'cancelled' where id = $1`, + watch.watchId + ); + + const response = await checkAction({ + request: request(token), + params: { watchId: watch.watchId }, + context: {}, + }); + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ code: "cancelled" }); + }); +}); + +describe("the agent's alert boundary", () => { + /** A second, plain member of the same organization. */ + async function seedMember(prisma: PrismaClient, seeded: Seeded) { + const member = await prisma.user.create({ + data: { + email: `member_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, + }); + return member; + } + + async function seedOutsider(prisma: PrismaClient) { + return prisma.user.create({ + data: { + email: `outsider_${Math.random().toString(36).slice(2, 10)}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + } + + async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { + return prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: `Watch alerts for ${email}`, + projectId: seeded.project.id, + alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email }, + deduplicationKey: `dashboard-agent-watch:${email}`, + }, + }); + } + + function listRequest(chatId: string) { + return { + request: new Request( + `https://app.trigger.dev/api/v1/dashboard-agent/alerts?chatId=${chatId}`, + { headers: { Authorization: "Bearer tr_uat_test" } } + ), + params: {}, + context: {} as never, + } as never; + } + + function createRequest(body: Record) { + return { + request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { + method: "POST", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: {}, + context: {} as never, + } as never; + } + + function deleteRequest(channelId: string, body: Record) { + return { + request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { + method: "DELETE", + headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, + body: JSON.stringify(body), + }), + params: { channelId }, + context: {} as never, + } as never; + } + + postgresTest( + "the dashboard lets any organization member manage a project's alerts", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-policy"); + const member = await seedMember(prisma, seeded); + const outsider = await seedOutsider(prisma); + + // The whole of the Alerts page's authorization, for list, create and delete alike. + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, member.id) + ).not.toBeNull(); + expect( + await findProjectBySlug(seeded.organization.slug, seeded.project.slug, outsider.id) + ).toBeNull(); + } + ); + + postgresTest( + "a plain member reads and writes watch alerts through the agent, an outsider reads nothing", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-member"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + await seedWatchChannel(prisma, seeded, member.email); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const listed = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(listed.status).toBe(200); + // The same channel the Alerts page would show this member. + expect((await listed.json()).alerts).toHaveLength(1); + + // An outsider has no chat here and no membership, so nothing resolves. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertsLoader(listRequest("chat_member"))) as Response; + expect(refused.status).toBe(404); + } + ); + + postgresTest( + "the agent only ever subscribes the caller's own address", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-create"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const own = (await alertsAction( + createRequest({ chatId: "chat_member", channel: "email" }) + )) as Response; + expect(own.status).toBe(200); + expect((await own.json()).target).toBe(member.email); + + // The Alerts page would let this member add anyone; the agent may not. + const other = (await alertsAction( + createRequest({ + chatId: "chat_member", + channel: "email", + email: "someone-else@example.com", + }) + )) as Response; + expect(other.status).toBe(400); + expect(await other.json()).toMatchObject({ code: "email_not_allowed" }); + + expect( + await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) + ).toBe(1); + } + ); + + postgresTest( + "the agent's delete only takes the watch type off a watch channel", + async ({ prisma, postgresContainer }) => { + await boot(prisma, postgresContainer.getConnectionUri()); + const seeded = await seed(prisma, "alert-delete"); + const member = await seedMember(prisma, seeded); + await createChat(ctx.agentDb, { + id: "chat_member", + organizationId: seeded.organization.id, + userId: member.id, + }); + const watchChannel = await seedWatchChannel(prisma, seeded, member.email); + + // A channel the agent never created and has no business touching. + const runAlerts = await prisma.projectAlertChannel.create({ + data: { + friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, + name: "Run failures", + projectId: seeded.project.id, + alertTypes: ["TASK_RUN"], + environmentTypes: ["PRODUCTION"], + type: "EMAIL", + properties: { email: member.email }, + }, + }); + + ctx.actor = { + userId: member.id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + + const removed = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(removed.status).toBe(200); + expect(await removed.json()).toMatchObject({ ok: true, disabledChannel: true }); + + // The Alerts page would let a member delete this outright; the agent gets a 404. + const untouched = (await alertChannelAction( + deleteRequest(runAlerts.id, { chatId: "chat_member" }) + )) as Response; + expect(untouched.status).toBe(404); + expect( + await prisma.projectAlertChannel.findFirst({ where: { id: runAlerts.id } }) + ).toMatchObject({ enabled: true, alertTypes: ["TASK_RUN"] }); + + // An outsider can't reach the channel at all. + ctx.actor = { + userId: (await seedOutsider(prisma)).id, + client: "dashboard-agent", + environmentId: seeded.environment.id, + }; + const refused = (await alertChannelAction( + deleteRequest(watchChannel.id, { chatId: "chat_member" }) + )) as Response; + expect(refused.status).toBe(404); + } + ); +}); diff --git a/apps/webapp/test/dashboardAgentWatches.test.ts b/apps/webapp/test/dashboardAgentWatches.test.ts deleted file mode 100644 index 7cfccf0a2..000000000 --- a/apps/webapp/test/dashboardAgentWatches.test.ts +++ /dev/null @@ -1,3421 +0,0 @@ -import { - appendChatMessageOnce, - armWatchBatch, - cancelWatch, - chatExists, - claimWatchBatchTick, - claimWatchDelivery, - claimWatchTick, - getWatchSubmission, - listActiveWatchesForBatch, - listWatchBatchGroupsToArm, - stopWatchBatch, - countUnreadWatchWakes, - countUserMessages, - createChat, - createDashboardAgentDb, - getChatMessages, - getWatch, - listActiveWatchesForChat, - listChatIdsWithUnreadWakes, - listRecentWatchWakes, - markWatchDelivered, - readWatchWakeFeed, - recordWatchCheck, - recordWatchSubmissionOutcome, - releaseWatchDelivery, - transitionWatchCondition, - WATCH_DELIVERY_CLAIM_STALE_MS, - type DashboardAgentDb, - type DashboardAgentDbClient, - type Watch, -} from "@internal/dashboard-agent-db"; -import { applyDashboardAgentMigrations } from "@internal/dashboard-agent-db/testing"; -import type { WatchDraft, WatchSpec } from "@internal/dashboard-agent-contracts"; -import { postgresTest } from "@internal/testcontainers"; -import type { PrismaClient } from "@trigger.dev/database"; -import { afterEach, beforeEach, describe, expect, vi } from "vitest"; -import { - previousCheckFacts, - type WatchCheckDeps, - type WatchRunRow, -} from "~/services/dashboardAgentWatchChecks"; - -// Every test here boots a container and replays the migrations inside its own budget, -// which does not fit vitest's 5s default on a loaded CI host. -vi.setConfig({ testTimeout: 60_000 }); - -const ctx = vi.hoisted(() => ({ - prisma: undefined as unknown as PrismaClient, - agentDb: undefined as unknown as DashboardAgentDb, - canAccess: true, - actor: undefined as undefined | { userId: string; client?: string; environmentId?: string }, - /** Every task id the suite would have triggered for real. */ - triggered: [] as string[], -})); - -vi.mock("~/services/uatRoutePreamble.server", () => ({ - authenticateUatOrApiRequest: async () => - ctx.actor - ? { - authenticationResult: { - type: "personalAccessToken", - result: { userId: ctx.actor.userId }, - }, - userActor: ctx.actor, - } - : undefined, -})); - -vi.mock("~/db.server", () => { - const proxy = new Proxy( - {}, - { get: (_target, prop) => (ctx.prisma as unknown as Record)[prop as string] } - ); - return { prisma: proxy, $replica: proxy, sqlDatabaseSchema: undefined }; -}); - -vi.mock("~/services/dashboardAgentDb.server", () => ({ - get dashboardAgentDb() { - return ctx.agentDb; - }, -})); - -vi.mock("~/v3/canAccessDashboardAgent.server", () => ({ - canAccessDashboardAgent: async () => ctx.canAccess, -})); - -// The routes drive the real service, which builds a TriggerClient from .env โ€” so an unmocked -// suite triggers actual runs against whatever origin .env names. -vi.mock("@trigger.dev/sdk", async (importOriginal) => { - const actual = await importOriginal>(); - return { - ...actual, - TriggerClient: class { - tasks = { - trigger: async (taskId: string) => { - ctx.triggered.push(taskId); - return { id: "run_test" }; - }, - }; - }, - }; -}); - -const SESSION_SECRET = "test-session-secret-for-watch-tokens"; -process.env.SESSION_SECRET = SESSION_SECRET; -// The agent's subscribe endpoint refuses without an email transport configured. -process.env.ALERT_FROM_EMAIL = "alerts@example.com"; -process.env.ALERT_EMAIL_TRANSPORT = "smtp"; -// Arming a batch chain builds a (stubbed) client only when this is set; unset in CI, it would -// no-op and the check test's trigger assertion would never see the batch task. -process.env.DASHBOARD_AGENT_SECRET_KEY = "test-dashboard-agent-secret"; - -const { - armDashboardAgentWatchBatch, - authorizeWatchEnvironment, - cancelDashboardAgentWatch, - createDashboardAgentWatch, - deleteChatWithWatches, - listActiveWatchesForChats, - submitDashboardAgentWatch, - watchBatchStaleMs, -} = await import("~/services/dashboardAgentWatches.server"); -const { action: checkAction } = - await import("~/routes/api.v1.dashboard-agent.watches.$watchId.check"); -const { action: createAction } = await import("~/routes/api.v1.dashboard-agent.watches"); -const { action: batchCheckAction } = - await import("~/routes/api.v1.dashboard-agent.watches.batch-check"); -const { - rearmDashboardAgentWatchBatches, - sweepDashboardAgentWatches, - WATCH_DELIVERY_GRACE_MS, - WATCH_EXPIRY_GRACE_MS, -} = await import("~/services/dashboardAgentWatchSweep.server"); -const { runWatchBatchCheck } = await import("~/services/dashboardAgentWatchBatch.server"); -const { signDashboardAgentWatchBatchToken, signDashboardAgentWatchToken } = - await import("~/services/dashboardAgentWatchToken.server"); -const { loader: alertsLoader, action: alertsAction } = - await import("~/routes/api.v1.dashboard-agent.alerts"); -const { action: alertChannelAction } = - await import("~/routes/api.v1.dashboard-agent.alerts.$channelId"); -const { findProjectBySlug } = await import("~/models/project.server"); -const { DASHBOARD_AGENT_WATCH_ALERT_TYPE, subscribeUserToWatchAlerts } = - await import("~/services/dashboardAgentWatchAlerts.server"); - -let agentDbClient: DashboardAgentDbClient | undefined; - -async function boot(prisma: PrismaClient, connectionUri: string) { - ctx.prisma = prisma; - await applyDashboardAgentMigrations((statement) => prisma.$executeRawUnsafe(statement)); - // A pool, not a single connection: the concurrent-create test needs the advisory lock to span connections. - agentDbClient = createDashboardAgentDb(connectionUri, { max: 8 }); - ctx.agentDb = agentDbClient.db; -} - -async function seed(prisma: PrismaClient, slugBase: string) { - const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; - const user = await prisma.user.create({ - data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, - }); - const organization = await prisma.organization.create({ data: { title: slug, slug } }); - await prisma.orgMember.create({ - data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, - }); - const project = await prisma.project.create({ - data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, - }); - const environment = await prisma.runtimeEnvironment.create({ - data: { - slug: "prod", - type: "PRODUCTION", - projectId: project.id, - organizationId: organization.id, - apiKey: `tr_prod_${slug}`, - pkApiKey: `pk_prod_${slug}`, - shortcode: `p${slug.slice(0, 6)}`, - }, - }); - return { user, organization, project, environment }; -} - -type Seeded = Awaited>; - -function authenticated(seeded: Seeded) { - return { - id: seeded.environment.id, - organizationId: seeded.organization.id, - projectId: seeded.project.id, - slug: "prod", - type: "PRODUCTION", - project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, - organization: { id: seeded.organization.id, slug: seeded.organization.slug }, - } as any; -} - -async function seedChat(seeded: Seeded, chatId = "chat_1") { - await createChat(ctx.agentDb, { - id: chatId, - organizationId: seeded.organization.id, - userId: seeded.user.id, - }); - return chatId; -} - -function runRow(overrides: Partial = {}): WatchRunRow { - return { - friendlyId: "run_1", - status: "PENDING", - queue: "task/my-task", - createdAt: new Date(), - queuedAt: null, - startedAt: null, - completedAt: null, - delayUntil: null, - ...overrides, - }; -} - -/** Injected readers. Defaults keep every condition pending with a live target. */ -function fakeCheckDeps(overrides: Partial = {}): WatchCheckDeps { - return { - readRun: async () => runRow(), - queueExists: async () => true, - readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), - readQueueOldestAge: async () => ({ ageMs: 30_000, source: "live_queue", current: true }), - readErrorRecurrence: async () => null, - readHealth: async () => ({ trustworthy: true, severity: "warn" }), - ...overrides, - }; -} - -const RUN_START: WatchSpec = { - kind: "run_start", - runId: "run_1", - checkEveryMinutes: 1, - maxHours: 2, - note: "tell me when it starts", -}; - -const BACKLOG: WatchSpec = { - kind: "backlog_drain", - queue: "task/my-task", - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me when it drains", -}; - -/** A run that exists for target validation and is gone when the immediate check reads it. */ -function readRunOnce(first: WatchRunRow) { - let calls = 0; - return async () => (calls++ === 0 ? first : null); -} - -function create(args: { - seeded: Seeded; - spec?: WatchSpec; - chatId?: string; - environmentId?: string; - investigateOnAttention?: boolean; - watchId?: string; - checkDeps?: Partial; - scheduled?: Array<{ watchId: string; token: string; tick: number }>; - onSchedule?: () => void; -}) { - const environment = authenticated(args.seeded); - return createDashboardAgentWatch({ - environment: args.environmentId ? { ...environment, id: args.environmentId } : environment, - userId: args.seeded.user.id, - chatId: args.chatId ?? "chat_1", - spec: args.spec ?? RUN_START, - investigateOnAttention: args.investigateOnAttention, - watchId: args.watchId, - deps: { - configured: () => true, - checkDeps: () => fakeCheckDeps(args.checkDeps), - scheduleTick: async (params) => { - args.onSchedule?.(); - args.scheduled?.push({ - watchId: params.watchId, - token: params.token, - tick: params.tick, - }); - }, - }, - }); -} - -beforeEach(() => { - ctx.canAccess = true; - ctx.actor = undefined; -}); - -afterEach(async () => { - await agentDbClient?.close(); - agentDbClient = undefined; -}); - -describe("createDashboardAgentWatch", () => { - postgresTest( - "creates an active watch and schedules its first tick", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const scheduled: Array<{ watchId: string; token: string; tick: number }> = []; - const result = await create({ seeded, scheduled }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.status).toBe("active"); - expect(result.identity).toBe("run_start:run_1"); - expect(result.immediate).toBeUndefined(); - - expect(scheduled).toHaveLength(1); - expect(scheduled[0]!.watchId).toBe(result.watchId); - expect(scheduled[0]!.tick).toBe(1); - expect(scheduled[0]!.token.startsWith("tr_daw_")).toBe(true); - - const row = await getWatch(ctx.agentDb, { id: result.watchId }); - expect(row).toMatchObject({ - status: "active", - deliveryStatus: "not_required", - environmentId: seeded.environment.id, - projectId: seeded.project.id, - organizationId: seeded.organization.id, - userId: seeded.user.id, - tickCount: 0, - investigateOnAttention: false, - projectRef: seeded.project.externalRef, - }); - } - ); - - postgresTest( - "records the investigate-on-attention consent when the caller asks for it", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ seeded, investigateOnAttention: true }); - - expect(result.ok).toBe(true); - if (!result.ok || !result.watching) return; - const row = await getWatch(ctx.agentDb, { id: result.watchId }); - expect(row?.investigateOnAttention).toBe(true); - expect(result.identity).toBe("run_start:run_1"); - } - ); - - postgresTest( - "stamps a server-set `since` on an error_recurrence watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const before = Date.now(); - const result = await create({ - seeded, - spec: { - kind: "error_recurrence", - fingerprint: "fp_1", - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me if it comes back", - }, - }); - expect(result.ok).toBe(true); - if (!result.ok) return; - - const row = await getWatch(ctx.agentDb, { id: result.watchId }); - const since = (row?.spec as { since?: string } | undefined)?.since; - expect(since).toBeDefined(); - expect(new Date(since!).getTime()).toBeGreaterThanOrEqual(before - 1000); - } - ); - - postgresTest( - "answers with a one-shot result and writes no row when the condition already holds", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - let ticks = 0; - const result = await create({ - seeded, - checkDeps: { - readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), - }, - onSchedule: () => { - ticks += 1; - }, - }); - - expect(result.ok).toBe(true); - if (!result.ok || result.watching) throw new Error("expected a one-shot result"); - expect(result.immediate.result).toBe("satisfied"); - expect(result.immediate.observed).toMatchObject({ kind: "run_start", started: true }); - expect(ticks).toBe(0); - - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - expect( - await listActiveWatchesForChats({ - chatIds: ["chat_1"], - organizationId: seeded.organization.id, - userId: seeded.user.id, - }) - ).toEqual({}); - } - ); - - postgresTest( - "answers with a one-shot result when the condition can no longer happen", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ - seeded, - checkDeps: { readRun: readRunOnce(runRow({ status: "QUEUED" })) }, - }); - - expect(result.ok).toBe(true); - if (!result.ok || result.watching) throw new Error("expected a one-shot result"); - expect(result.immediate.result).toBe("terminal_unsatisfied"); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses a duplicate before running the immediate check", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const first = await create({ seeded }); - expect(first.ok).toBe(true); - - let checks = 0; - const second = await create({ - seeded, - checkDeps: { - readRun: async () => { - checks += 1; - return runRow({ status: "EXECUTING", startedAt: new Date() }); - }, - }, - }); - - expect(second).toMatchObject({ ok: false, code: "duplicate" }); - expect(checks).toBe(1); - } - ); - - postgresTest( - "cancels the row silently when the first tick can't be scheduled", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ - seeded, - onSchedule: () => { - throw new Error("no agent project"); - }, - }); - - expect(result).toMatchObject({ ok: false, code: "internal" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - const rows = await ctx.prisma.$queryRawUnsafe< - { status: string; cancel_reason: string; delivery_status: string }[] - >( - `select status, cancel_reason, delivery_status - from trigger_dashboard_agent.watches where chat_id = 'chat_1'` - ); - expect(rows).toMatchObject([ - { - status: "cancelled", - cancel_reason: "scheduling_failed", - delivery_status: "not_required", - }, - ]); - } - ); - - postgresTest( - "rejects a target that doesn't exist, writing nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const result = await create({ - seeded, - spec: BACKLOG, - checkDeps: { queueExists: async () => false }, - }); - - expect(result).toMatchObject({ ok: false, code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "dedups the same condition and allows it in another environment", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - const first = await create({ seeded }); - expect(first.ok).toBe(true); - - const second = await create({ seeded }); - expect(second).toMatchObject({ ok: false, code: "duplicate" }); - if (!second.ok && first.ok) expect(second.existingId).toBe(first.watchId); - - const otherEnv = await prisma.runtimeEnvironment.create({ - data: { - slug: "stg", - type: "STAGING", - projectId: seeded.project.id, - organizationId: seeded.organization.id, - apiKey: `tr_stg_${seeded.project.slug}`, - pkApiKey: `pk_stg_${seeded.project.slug}`, - shortcode: `s${seeded.project.slug.slice(0, 6)}`, - }, - }); - const third = await create({ seeded, environmentId: otherEnv.id }); - expect(third.ok).toBe(true); - } - ); - - postgresTest( - "refuses a 4th active watch in the same chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "watch"); - await seedChat(seeded); - - for (const runId of ["run_1", "run_2", "run_3"]) { - const created = await create({ seeded, spec: { ...RUN_START, runId } }); - expect(created.ok).toBe(true); - } - - const fourth = await create({ seeded, spec: { ...RUN_START, runId: "run_4" } }); - expect(fourth).toMatchObject({ ok: false, code: "limit_reached" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); - } - ); - - postgresTest( - "holds the โ‰ค3 limit against four concurrent creates", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "race"); - await seedChat(seeded); - - const results = await Promise.all( - ["run_1", "run_2", "run_3", "run_4"].map((runId) => - create({ seeded, spec: { ...RUN_START, runId } }) - ) - ); - - expect(results.filter((result) => result.ok)).toHaveLength(3); - expect( - results.filter((result) => !result.ok && result.code === "limit_reached") - ).toHaveLength(1); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(3); - } - ); -}); - -describe("the createWatch endpoint's authorization", () => { - function post(body: unknown) { - return createAction({ - request: new Request("https://example.com/api/v1/dashboard-agent/watches", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(body), - }), - params: {}, - context: {}, - }); - } - - const validBody = (chatId: string) => ({ spec: RUN_START, chatId }); - - postgresTest("401s without a delegated token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const response = await post(validBody("chat_1")); - expect(response.status).toBe(401); - }); - - postgresTest("403s for any other client's token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "adapter"); - ctx.actor = { userId: seeded.user.id, client: "cli", environmentId: seeded.environment.id }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "forbidden_client" }); - }); - - postgresTest( - "refuses a chat the authenticated user doesn't own, writing nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const owner = await seed(prisma, "owner"); - const stranger = await seed(prisma, "stranger"); - await createChat(ctx.agentDb, { - id: "chat_victim", - organizationId: owner.organization.id, - userId: owner.user.id, - }); - - ctx.actor = { - userId: stranger.user.id, - client: "dashboard-agent", - environmentId: stranger.environment.id, - }; - - const response = await post(validBody("chat_victim")); - expect(response.status).toBe(404); - expect(await response.json()).toMatchObject({ code: "chat_not_found" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_victim" })).toHaveLength( - 0 - ); - } - ); - - postgresTest( - "refuses a token with no environment scope", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "noscope"); - await seedChat(seeded, "chat_1"); - ctx.actor = { userId: seeded.user.id, client: "dashboard-agent" }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses a body naming a different environment than the token's", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "mismatch"); - const other = await seed(prisma, "othermismatch"); - await seedChat(seeded, "chat_1"); - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const response = await post({ - ...validBody("chat_1"), - environmentId: other.environment.id, - }); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "binds to the token's environment, not the chat's stored context", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "binding"); - const otherProject = await prisma.project.create({ - data: { - name: `${seeded.project.slug}_b`, - slug: `${seeded.project.slug}_b`, - organizationId: seeded.organization.id, - externalRef: `proj_${seeded.project.slug}_b`, - }, - }); - const otherEnvironment = await prisma.runtimeEnvironment.create({ - data: { - slug: "prod", - type: "PRODUCTION", - projectId: otherProject.id, - organizationId: seeded.organization.id, - apiKey: `tr_prod_${otherProject.slug}`, - pkApiKey: `pk_prod_${otherProject.slug}`, - shortcode: `b${otherProject.slug.slice(0, 6)}`, - }, - }); - - await createChat(ctx.agentDb, { - id: "chat_1", - organizationId: seeded.organization.id, - userId: seeded.user.id, - metadata: { - context: { - environmentId: seeded.environment.id, - projectRef: seeded.project.externalRef, - }, - }, - }); - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: otherEnvironment.id, - }; - - const response = await post({ - ...validBody("chat_1"), - projectRef: seeded.project.externalRef, - }); - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ code: "environment_mismatch" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - postgresTest( - "refuses an environment in another org than the chat's", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "crossorg"); - const other = await seed(prisma, "otherorg"); - await prisma.orgMember.create({ - data: { - organizationId: other.organization.id, - userId: seeded.user.id, - role: "ADMIN", - }, - }); - await seedChat(seeded, "chat_1"); - - ctx.actor = { - userId: seeded.user.id, - client: "dashboard-agent", - environmentId: other.environment.id, - }; - - const response = await post(validBody("chat_1")); - expect(response.status).toBe(404); - expect(await response.json()).toMatchObject({ code: "invalid_target" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); -}); - -describe("the chat cascade and the list view", () => { - postgresTest( - "deleting a chat soft-deletes it and cancels its active watches in one call", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "cascade"); - await seedChat(seeded, "chat_1"); - await seedChat(seeded, "chat_2"); - - const mine = await create({ seeded, chatId: "chat_1" }); - const theirs = await create({ seeded, chatId: "chat_2" }); - expect(mine.ok && theirs.ok).toBe(true); - if (!mine.ok || !theirs.ok) return; - - expect( - await deleteChatWithWatches({ - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) - ).toEqual({ - deleted: true, - cancelledWatches: 1, - }); - - expect( - await chatExists(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) - ).toBe(false); - expect(await getWatch(ctx.agentDb, { id: mine.watchId })).toMatchObject({ - status: "cancelled", - cancelReason: "chat_deleted", - deliveryStatus: "not_required", - }); - expect(await getWatch(ctx.agentDb, { id: theirs.watchId })).toMatchObject({ - status: "active", - }); - } - ); - - postgresTest( - "a user's own cancel leaves one neutral line in the chat, and only one", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "usercancel"); - await seedChat(seeded, "chat_1"); - - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - const cancel = () => - cancelDashboardAgentWatch({ - watchId: created.watchId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - - expect(await cancel()).toMatchObject({ - cancelled: true, - messages: [ - { - id: `watch-cancelled:${created.watchId}`, - role: "assistant", - parts: [{ type: "text", text: "Stopped watching run run_1." }], - }, - ], - }); - expect(await getWatch(ctx.agentDb, { id: created.watchId })).toMatchObject({ - status: "cancelled", - cancelReason: "user", - deliveryStatus: "not_required", - }); - expect(await storedMessages(seeded, "chat_1")).toMatchObject([ - { id: `watch-cancelled:${created.watchId}`, role: "assistant" }, - ]); - - // The row is no longer active, so the second cancel writes nothing at all. - expect(await cancel()).toEqual({ cancelled: false, messages: [] }); - expect(await storedMessages(seeded, "chat_1")).toHaveLength(1); - } - ); - - postgresTest( - "a chat delete cancels its watches without a line in the chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "silentcancel"); - await seedChat(seeded, "chat_1"); - - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - - await deleteChatWithWatches({ - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - - const rows = await ctx.prisma.$queryRawUnsafe<{ message_id: string }[]>( - `select message_id from trigger_dashboard_agent.chat_messages where chat_id = 'chat_1'` - ); - expect(rows).toEqual([]); - } - ); - - postgresTest( - "aggregates active watches per chat in one query", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "chips"); - await seedChat(seeded, "chat_1"); - await seedChat(seeded, "chat_2"); - - const a = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_1" } }); - const b = await create({ seeded, chatId: "chat_1", spec: { ...RUN_START, runId: "run_2" } }); - const c = await create({ seeded, chatId: "chat_2" }); - expect(a.ok && b.ok && c.ok).toBe(true); - - const byChat = await listActiveWatchesForChats({ - chatIds: ["chat_1", "chat_2", "chat_missing"], - organizationId: seeded.organization.id, - userId: seeded.user.id, - }); - expect(byChat.chat_1).toHaveLength(2); - expect(byChat.chat_2).toHaveLength(1); - expect(byChat.chat_missing).toBeUndefined(); - expect(byChat.chat_2![0]).toMatchObject({ - identity: "run_start:run_1", - status: "active", - kind: "run_start", - note: RUN_START.note, - }); - - if (a.ok) await cancelWatch(ctx.agentDb, { id: a.watchId, reason: "user" }); - if (b.ok) await cancelWatch(ctx.agentDb, { id: b.watchId, reason: "user" }); - expect( - ( - await listActiveWatchesForChats({ - chatIds: ["chat_1"], - organizationId: seeded.organization.id, - userId: seeded.user.id, - }) - ).chat_1 - ).toBeUndefined(); - } - ); - - postgresTest("returns nothing for an empty chat list", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - expect( - await listActiveWatchesForChats({ chatIds: [], organizationId: "org_x", userId: "user_x" }) - ).toEqual({}); - }); -}); - -describe("unread watch wakes", () => { - postgresTest( - "only signals a wake once its delivery landed", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "unread"); - await seedChat(seeded, "chat_1"); - - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - const scope = { organizationId: seeded.organization.id, userId: seeded.user.id }; - const recent = { ...scope, deliveredAfter: new Date(Date.now() - 15 * 60 * 1000) }; - - if (!created.watching) throw new Error("expected a watch"); - await transitionWatchCondition(ctx.agentDb, { - id: created.watchId, - resolution: "condition_met", - }); - expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(0); - expect(await listRecentWatchWakes(ctx.agentDb, recent)).toEqual([]); - expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set()); - - await markWatchDelivered(ctx.agentDb, { id: created.watchId }); - expect(await countUnreadWatchWakes(ctx.agentDb, scope)).toBe(1); - expect(await listRecentWatchWakes(ctx.agentDb, recent)).toMatchObject([ - { watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }, - ]); - expect(await listChatIdsWithUnreadWakes(ctx.agentDb, scope)).toEqual(new Set(["chat_1"])); - - // The poll's single query answers both halves the same way. - expect(await readWatchWakeFeed(ctx.agentDb, recent)).toMatchObject({ - unreadWakes: 1, - wakes: [{ watchId: created.watchId, chatId: "chat_1", outcome: "fired", unread: true }], - }); - - // An unread wake from before the window still counts, but isn't narrated again. - expect( - await readWatchWakeFeed(ctx.agentDb, { - ...scope, - deliveredAfter: new Date(Date.now() + 60_000), - }) - ).toMatchObject({ unreadWakes: 1, wakes: [] }); - } - ); -}); - -describe("authorizeWatchEnvironment", () => { - postgresTest( - "passes for a member and fails once membership is gone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "auth"); - - const params = { - userId: seeded.user.id, - organizationId: seeded.organization.id, - projectId: seeded.project.id, - environmentId: seeded.environment.id, - }; - - expect((await authorizeWatchEnvironment(params)).ok).toBe(true); - - await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); - expect(await authorizeWatchEnvironment(params)).toEqual({ - ok: false, - reason: "access_revoked", - }); - } - ); - - postgresTest("fails when the feature gate is revoked", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "auth"); - ctx.canAccess = false; - - expect( - await authorizeWatchEnvironment({ - userId: seeded.user.id, - organizationId: seeded.organization.id, - projectId: seeded.project.id, - environmentId: seeded.environment.id, - }) - ).toEqual({ ok: false, reason: "access_revoked" }); - }); - - postgresTest( - "fails when the snapshot names a different project", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "auth"); - const other = await seed(prisma, "other"); - - expect( - await authorizeWatchEnvironment({ - userId: seeded.user.id, - organizationId: seeded.organization.id, - projectId: other.project.id, - environmentId: seeded.environment.id, - }) - ).toEqual({ ok: false, reason: "access_revoked" }); - } - ); -}); - -describe("the watch sweep", () => { - async function overdueWatch(seeded: Seeded, chatId = "chat_1") { - const created = await create({ seeded, chatId }); - if (!created.ok) throw new Error("the watch wasn't created"); - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 hour' where id = $1`, - created.watchId - ); - return created.watchId; - } - - function sweepDeps(args: { - seeded: Seeded; - checkDeps?: Partial; - revoked?: boolean; - now?: Date; - failDelivery?: boolean; - delivered: string[]; - }) { - return { - now: () => args.now ?? new Date(), - checkDeps: () => fakeCheckDeps(args.checkDeps), - authorize: async () => - args.revoked - ? ({ ok: false, reason: "access_revoked" } as const) - : ({ ok: true, environment: authenticated(args.seeded) } as const), - deliver: async (watch: Watch) => { - if (args.failDelivery) throw new Error("the delivery couldn't be scheduled"); - args.delivered.push(watch.id); - }, - configured: () => true, - }; - } - - postgresTest( - "runs the final check on an overdue watch and fires it at the buzzer", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches( - sweepDeps({ - seeded, - delivered, - checkDeps: { - readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }), - }, - }) - ); - - expect(result).toMatchObject({ overdue: 1, fired: 1, expired: 0, cancelled: 0, failed: 0 }); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "fired", - deliveryStatus: "pending", - }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "expires an overdue watch the check says hasn't happened, as verified", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); - - expect(result).toMatchObject({ overdue: 1, expired: 1, failed: 0 }); - const row = await getWatch(ctx.agentDb, { id: watchId }); - expect(row).toMatchObject({ status: "expired", deliveryStatus: "pending" }); - expect(row?.lastResult).toMatchObject({ verified: true, reason: "not_met_by_expiry" }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "cancels an overdue watch whose user lost access, and never wakes the chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches( - sweepDeps({ seeded, delivered, revoked: true }) - ); - - expect(result).toMatchObject({ overdue: 1, cancelled: 1, expired: 0, fired: 0, failed: 0 }); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "cancelled", - cancelReason: "access_revoked", - deliveryStatus: "not_required", - }); - expect(delivered).toEqual([]); - } - ); - - postgresTest( - "leaves a watch that is still inside its deadline alone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const created = await create({ seeded }); - expect(created.ok).toBe(true); - - const delivered: string[] = []; - const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered })); - - expect(result).toMatchObject({ overdue: 0, undelivered: 0 }); - expect(delivered).toEqual([]); - } - ); - - postgresTest( - "recovers a wake the delivery lost, through the real query, exactly once", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - await expect( - sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, failDelivery: true })) - ).rejects.toThrow(/failed on 1 watches/); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "expired", - deliveryStatus: "pending", - }); - expect(delivered).toEqual([]); - - const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); - const second = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); - expect(second).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); - expect(delivered).toEqual([watchId]); - - await markWatchDelivered(ctx.agentDb, { id: watchId }); - const third = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); - expect(third).toMatchObject({ undelivered: 0, redelivered: 0 }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "a deliverer that died mid-delivery is recovered, but a fresh claim is left alone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] })); - - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches - set delivery_status = 'delivering', - delivery_claimed_at = now(), - last_checked_at = now() - interval '1 hour' - where id = $1`, - watchId - ); - expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: [] }))).toMatchObject({ - undelivered: 0, - }); - - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches - set delivery_claimed_at = now() - interval '1 hour' where id = $1`, - watchId - ); - const recovered: string[] = []; - expect( - await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered: recovered })) - ).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); - expect(recovered).toEqual([watchId]); - } - ); - - postgresTest( - "leaves nothing owed for a request the immediate check already answered", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - - const created = await create({ - seeded, - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - expect(created.ok).toBe(true); - if (!created.ok || created.watching) throw new Error("expected a one-shot result"); - - const delivered: string[] = []; - const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); - const result = await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })); - - expect(result).toMatchObject({ overdue: 0, undelivered: 0, redelivered: 0 }); - expect(delivered).toEqual([]); - } - ); - - postgresTest( - "finalizes overdue watches even with no agent to deliver to, and delivers once it's back", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const watchId = await overdueWatch(seeded); - - const delivered: string[] = []; - const unconfigured = await sweepDashboardAgentWatches({ - ...sweepDeps({ seeded, delivered }), - configured: () => false, - }); - - expect(unconfigured).toMatchObject({ - overdue: 1, - expired: 1, - deliveryDeferred: 1, - undelivered: 0, - redelivered: 0, - failed: 0, - }); - expect(delivered).toEqual([]); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - status: "expired", - deliveryStatus: "pending", - }); - - const later = new Date(Date.now() + WATCH_DELIVERY_GRACE_MS + 60_000); - const restored = await sweepDashboardAgentWatches( - sweepDeps({ seeded, delivered, now: later }) - ); - expect(restored).toMatchObject({ undelivered: 1, redelivered: 1, failed: 0 }); - expect(delivered).toEqual([watchId]); - } - ); - - postgresTest( - "the expiry grace keeps the sweep off a watch the tick chain is still finishing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "sweep"); - await seedChat(seeded); - const created = await create({ seeded }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - // A second past the deadline, so the chain's own final check owns this window. - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 second' where id = $1`, - created.watchId - ); - const delivered: string[] = []; - expect(await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered }))).toMatchObject({ - overdue: 0, - }); - - const later = new Date(Date.now() + WATCH_EXPIRY_GRACE_MS + 60_000); - expect( - await sweepDashboardAgentWatches(sweepDeps({ seeded, delivered, now: later })) - ).toMatchObject({ overdue: 1, expired: 1 }); - } - ); -}); - -describe("the tick claim", () => { - postgresTest( - "claiming a generation is not an observation: only a recorded check stamps one", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim"); - await seedChat(seeded); - const created = await create({ seeded }); - expect(created.ok).toBe(true); - if (!created.ok) return; - - const claimed = await claimWatchTick(ctx.agentDb, { id: created.watchId, generation: 1 }); - expect(claimed).toMatchObject({ tickCount: 1, lastCheckedAt: null, lastResult: null }); - - await recordWatchCheck(ctx.agentDb, { id: created.watchId, lastResult: { pending: 4 } }); - const row = await getWatch(ctx.agentDb, { id: created.watchId }); - expect(row?.lastCheckedAt).toBeInstanceOf(Date); - expect(row?.lastResult).toMatchObject({ pending: 4 }); - expect(row?.tickCount).toBe(1); - } - ); -}); - -// The delivery claim's fencing token: a hung deliverer is taken over, so an unfenced release or mark would touch the new owner's claim. -describe("the delivery claim", () => { - async function firedWatch(seeded: Seeded) { - const created = await create({ seeded }); - expect(created.ok).toBe(true); - if (!created.ok) throw new Error("the watch wasn't created"); - const transitioned = await transitionWatchCondition(ctx.agentDb, { - id: created.watchId, - status: "fired", - lastResult: { result: "satisfied", facts: { verified: true } }, - }); - expect(transitioned).toMatchObject({ deliveryStatus: "pending" }); - return created.watchId; - } - - function staleBefore() { - return new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS); - } - - async function ageClaim(watchId: string) { - await ctx.prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches - set delivery_claimed_at = now() - interval '1 hour' where id = $1`, - watchId - ); - } - - postgresTest( - "a stale takeover makes the old owner's release a no-op, and the new owner delivers once", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim-fence"); - await seedChat(seeded); - const watchId = await firedWatch(seeded); - - const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(a).not.toBeNull(); - if (!a) return; - - await ageClaim(watchId); - const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(b).not.toBeNull(); - if (!b) return; - expect(b.claimId).not.toBe(a.claimId); - - expect( - await releaseWatchDelivery(ctx.agentDb, { id: watchId, claimId: a.claimId }) - ).toBeNull(); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - deliveryStatus: "delivering", - deliveryClaimId: b.claimId, - }); - - expect( - await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) - ).toBeNull(); - - expect( - await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) - ).toMatchObject({ deliveryStatus: "delivered" }); - expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId })).toBeNull(); - } - ); - - postgresTest( - "a late delivered-mark from the old owner completes nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim-late"); - await seedChat(seeded); - const watchId = await firedWatch(seeded); - - const a = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(a).not.toBeNull(); - if (!a) return; - await ageClaim(watchId); - const b = await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }); - expect(b).not.toBeNull(); - if (!b) return; - - expect(await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: a.claimId })).toBeNull(); - expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); - expect(await getWatch(ctx.agentDb, { id: watchId })).toMatchObject({ - deliveryStatus: "delivering", - deliveredAt: null, - }); - - expect( - await markWatchDelivered(ctx.agentDb, { id: watchId, claimId: b.claimId }) - ).toMatchObject({ deliveryStatus: "delivered" }); - } - ); - - postgresTest( - "the inline path marks a pending delivery without a claim", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "claim-inline"); - await seedChat(seeded); - const watchId = await firedWatch(seeded); - - expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toMatchObject({ - deliveryStatus: "delivered", - }); - expect(await markWatchDelivered(ctx.agentDb, { id: watchId })).toBeNull(); - expect( - await claimWatchDelivery(ctx.agentDb, { id: watchId, staleBefore: staleBefore() }) - ).toBeNull(); - } - ); -}); - -describe("deleting a chat while a watch is being created", () => { - postgresTest("holds in both orders", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "race"); - - for (const deleteFirst of [true, false]) { - const chatId = `chat_${deleteFirst ? "del" : "add"}`; - await seedChat(seeded, chatId); - - const creating = () => create({ seeded, chatId }); - const deleting = () => - deleteChatWithWatches({ - chatId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - const [a, b] = deleteFirst - ? await Promise.all([deleting(), creating()]) - : await Promise.all([creating(), deleting()]); - expect(a).toBeDefined(); - expect(b).toBeDefined(); - - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId })).toEqual([]); - expect( - await chatExists(ctx.agentDb, { - chatId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) - ).toBe(false); - } - }); - - postgresTest( - "refuses a create against an already-deleted chat", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "race"); - await seedChat(seeded); - await deleteChatWithWatches({ - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - - expect(await create({ seeded })).toMatchObject({ ok: false, code: "chat_not_found" }); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); - } - ); -}); - -describe("the check endpoint", () => { - function request(token: string, body: unknown = {}) { - return new Request("https://example.com/api/v1/dashboard-agent/watches/x/check", { - method: "POST", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: JSON.stringify(body), - }); - } - - async function activeWatch(seeded: Seeded, spec?: WatchSpec) { - const result = await create({ seeded, spec }); - if (!result.ok) throw new Error(`watch not created: ${result.code}`); - return result; - } - - function tokenFor(watchId: string, expiresAt: Date) { - return signDashboardAgentWatchToken(SESSION_SECRET, { watchId, expiresAt }); - } - - postgresTest("401s on a bad token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - - const response = await checkAction({ - request: request("tr_daw_nonsense"), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(401); - }); - - postgresTest("403s when the token names another watch", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor("watch_someone_else", watch.expiresAt); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "watch_mismatch" }); - }); - - postgresTest("answers a check and records what it saw", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(200); - const body = await response.json(); - expect(body.result).toBe("terminal_unsatisfied"); - - // Arming the chain goes through the stubbed client, never a real trigger. - expect(ctx.triggered).toContain("dashboard-agent-watch-batch"); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - expect(row?.lastCheckedAt).not.toBeNull(); - expect(row?.tickCount).toBe(0); - expect(row?.status).toBe("active"); - }); - - postgresTest( - "refuses an ordinary check after expiry but allows the final one in grace", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() - interval '1 minute' where id = $1`, - watch.watchId - ); - - const token = await tokenFor(watch.watchId, watch.expiresAt); - - const refused = await checkAction({ - request: request(token, {}), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(refused.status).toBe(403); - expect(await refused.json()).toMatchObject({ code: "expired" }); - - const allowed = await checkAction({ - request: request(token, { final: true }), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(allowed.status).toBe(200); - } - ); - - postgresTest( - "cancels the watch on revoked access, without reading environment data", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - await prisma.orgMember.deleteMany({ where: { userId: seeded.user.id } }); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "access_revoked" }); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - expect(row).toMatchObject({ - status: "cancelled", - cancelReason: "access_revoked", - deliveryStatus: "not_required", - }); - expect(row?.tickCount).toBe(0); - expect(row?.lastResult).toBeNull(); - } - ); - - postgresTest( - "a check that couldn't read anything leaves the row's last look and facts alone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - - // The queue exists, so the check gets past the target read and fails on the depth - // read: there is no live queue or analytics store behind this environment. - const queue = "task/stalling"; - await prisma.taskQueue.create({ - data: { - runtimeEnvironmentId: seeded.environment.id, - projectId: seeded.project.id, - name: queue, - friendlyId: `queue_${Math.random().toString(36).slice(2, 10)}`, - orderableName: queue, - }, - }); - - const watch = await activeWatch(seeded, { - kind: "queue_stalled", - queue, - ticks: 3, - checkEveryMinutes: 5, - maxHours: 6, - note: "tell me if the queue stops moving", - }); - - // Two no-progress checks already behind it, last looked at an hour ago. - const checkedAt = new Date(Date.now() - 60 * 60 * 1000); - await recordWatchCheck(ctx.agentDb, { - id: watch.watchId, - lastCheckedAt: checkedAt, - lastResult: { - result: "pending", - facts: { queue, depth: 412, notDecreasingStreak: 2, ticks: 3 }, - }, - }); - - const token = await tokenFor(watch.watchId, watch.expiresAt); - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ result: "unavailable" }); - - const row = await getWatch(ctx.agentDb, { id: watch.watchId }); - // Nothing was checked, so the watch is still due at the next tick. - expect(row?.lastCheckedAt?.getTime()).toBe(checkedAt.getTime()); - // And the streak the earlier ticks built is still there to be continued. - expect(previousCheckFacts(row?.lastResult)).toMatchObject({ - depth: 412, - notDecreasingStreak: 2, - }); - }, - 120_000 - ); - - postgresTest("403s once the watch is terminal", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "check"); - await seedChat(seeded); - const watch = await activeWatch(seeded); - const token = await tokenFor(watch.watchId, watch.expiresAt); - - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set status = 'cancelled' where id = $1`, - watch.watchId - ); - - const response = await checkAction({ - request: request(token), - params: { watchId: watch.watchId }, - context: {}, - }); - expect(response.status).toBe(403); - expect(await response.json()).toMatchObject({ code: "cancelled" }); - }); -}); - -/** A configured card, with both follow-ups off unless a test turns one on. */ -function draftFor(spec: WatchSpec, followUp: Partial = {}): WatchDraft { - return { - spec, - followUp: { investigateOnAttention: false, notifyExternally: false, ...followUp }, - }; -} - -function submit(args: { - seeded: Seeded; - draft?: WatchDraft; - chatId?: string; - clientRequestId?: string; - checkDeps?: Partial; - subscribed?: boolean; - /** Replaces the fake outright, so a test can hand the submit the real subscribe. */ - subscribe?: typeof subscribeUserToWatchAlerts; - onSchedule?: () => void; - /** Wraps the creation step, so a test can die at the exact point after it. */ - create?: typeof createDashboardAgentWatch; -}) { - return submitDashboardAgentWatch({ - environment: authenticated(args.seeded), - userId: args.seeded.user.id, - organizationId: args.seeded.organization.id, - chatId: args.chatId, - clientRequestId: args.clientRequestId ?? "wreq_1", - draft: args.draft ?? draftFor(RUN_START), - deps: { - configured: () => true, - checkDeps: () => fakeCheckDeps(args.checkDeps), - scheduleTick: async () => args.onSchedule?.(), - ...(args.create ? { create: args.create } : {}), - subscribe: - args.subscribe ?? - (async () => - args.subscribed === false - ? { ok: false, reason: "dashboard_agent_disabled" } - : { ok: true, email: args.seeded.user.email }), - }, - }); -} - -function storedMessages(seeded: Seeded, chatId: string) { - return getChatMessages(ctx.agentDb, { - chatId, - userId: seeded.user.id, - organizationId: seeded.organization.id, - }) as Promise | null>; -} - -/** - * The Alerts page authorizes with `findProjectBySlug` alone (see - * `_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx`): - * every organization member may list, create and delete a project's alert channels, with - * no role check. These tests pin that policy and prove the agent's routes never write - * wider than it. - */ -describe("the agent's alert boundary", () => { - /** A second, plain member of the same organization. */ - async function seedMember(prisma: PrismaClient, seeded: Seeded) { - const member = await prisma.user.create({ - data: { - email: `member_${Math.random().toString(36).slice(2, 10)}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - await prisma.orgMember.create({ - data: { organizationId: seeded.organization.id, userId: member.id, role: "MEMBER" }, - }); - return member; - } - - async function seedOutsider(prisma: PrismaClient) { - return prisma.user.create({ - data: { - email: `outsider_${Math.random().toString(36).slice(2, 10)}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - } - - async function seedWatchChannel(prisma: PrismaClient, seeded: Seeded, email: string) { - return prisma.projectAlertChannel.create({ - data: { - friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, - name: `Watch alerts for ${email}`, - projectId: seeded.project.id, - alertTypes: [DASHBOARD_AGENT_WATCH_ALERT_TYPE as never], - environmentTypes: ["PRODUCTION"], - type: "EMAIL", - properties: { email }, - deduplicationKey: `dashboard-agent-watch:${email}`, - }, - }); - } - - function listRequest(chatId: string) { - return { - request: new Request( - `https://app.trigger.dev/api/v1/dashboard-agent/alerts?chatId=${chatId}`, - { headers: { Authorization: "Bearer tr_uat_test" } } - ), - params: {}, - context: {} as never, - } as never; - } - - function createRequest(body: Record) { - return { - request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/alerts", { - method: "POST", - headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, - body: JSON.stringify(body), - }), - params: {}, - context: {} as never, - } as never; - } - - function deleteRequest(channelId: string, body: Record) { - return { - request: new Request(`https://app.trigger.dev/api/v1/dashboard-agent/alerts/${channelId}`, { - method: "DELETE", - headers: { Authorization: "Bearer tr_uat_test", "content-type": "application/json" }, - body: JSON.stringify(body), - }), - params: { channelId }, - context: {} as never, - } as never; - } - - postgresTest( - "the dashboard lets any organization member manage a project's alerts", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-policy"); - const member = await seedMember(prisma, seeded); - const outsider = await seedOutsider(prisma); - - // The whole of the Alerts page's authorization, for list, create and delete alike. - expect( - await findProjectBySlug(seeded.organization.slug, seeded.project.slug, member.id) - ).not.toBeNull(); - expect( - await findProjectBySlug(seeded.organization.slug, seeded.project.slug, outsider.id) - ).toBeNull(); - } - ); - - postgresTest( - "a plain member reads and writes watch alerts through the agent, an outsider reads nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-member"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - await seedWatchChannel(prisma, seeded, member.email); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const listed = (await alertsLoader(listRequest("chat_member"))) as Response; - expect(listed.status).toBe(200); - // The same channel the Alerts page would show this member. - expect((await listed.json()).alerts).toHaveLength(1); - - // An outsider has no chat here and no membership, so nothing resolves. - ctx.actor = { - userId: (await seedOutsider(prisma)).id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const refused = (await alertsLoader(listRequest("chat_member"))) as Response; - expect(refused.status).toBe(404); - } - ); - - postgresTest( - "the agent only ever subscribes the caller's own address", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-create"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const own = (await alertsAction( - createRequest({ chatId: "chat_member", channel: "email" }) - )) as Response; - expect(own.status).toBe(200); - expect((await own.json()).target).toBe(member.email); - - // The Alerts page would let this member add anyone; the agent may not. - const other = (await alertsAction( - createRequest({ - chatId: "chat_member", - channel: "email", - email: "someone-else@example.com", - }) - )) as Response; - expect(other.status).toBe(400); - expect(await other.json()).toMatchObject({ code: "email_not_allowed" }); - - expect( - await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) - ).toBe(1); - } - ); - - postgresTest( - "the agent's delete only takes the watch type off a watch channel", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "alert-delete"); - const member = await seedMember(prisma, seeded); - await createChat(ctx.agentDb, { - id: "chat_member", - organizationId: seeded.organization.id, - userId: member.id, - }); - const watchChannel = await seedWatchChannel(prisma, seeded, member.email); - - // A channel the agent never created and has no business touching. - const runAlerts = await prisma.projectAlertChannel.create({ - data: { - friendlyId: `alert_${Math.random().toString(36).slice(2, 10)}`, - name: "Run failures", - projectId: seeded.project.id, - alertTypes: ["TASK_RUN"], - environmentTypes: ["PRODUCTION"], - type: "EMAIL", - properties: { email: member.email }, - }, - }); - - ctx.actor = { - userId: member.id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - - const removed = (await alertChannelAction( - deleteRequest(watchChannel.id, { chatId: "chat_member" }) - )) as Response; - expect(removed.status).toBe(200); - expect(await removed.json()).toMatchObject({ ok: true, disabledChannel: true }); - - // The Alerts page would let a member delete this outright; the agent gets a 404. - const untouched = (await alertChannelAction( - deleteRequest(runAlerts.id, { chatId: "chat_member" }) - )) as Response; - expect(untouched.status).toBe(404); - expect( - await prisma.projectAlertChannel.findFirst({ where: { id: runAlerts.id } }) - ).toMatchObject({ enabled: true, alertTypes: ["TASK_RUN"] }); - - // An outsider can't reach the channel at all. - ctx.actor = { - userId: (await seedOutsider(prisma)).id, - client: "dashboard-agent", - environmentId: seeded.environment.id, - }; - const refused = (await alertChannelAction( - deleteRequest(watchChannel.id, { chatId: "chat_member" }) - )) as Response; - expect(refused.status).toBe(404); - } - ); -}); - -describe("the watch card submit", () => { - postgresTest( - "records what the user confirmed before the watch, and confirms it after", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit"); - await seedChat(seeded); - - const result = await submit({ - seeded, - chatId: "chat_1", - draft: draftFor(RUN_START, { investigateOnAttention: true }), - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(true); - expect(result.repaired).toBe(false); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${result.watchId}`, - ]); - // The consent record is the user's, and it states the condition and the lifetime. - expect(stored?.[0]).toMatchObject({ role: "user" }); - expect(JSON.stringify(stored?.[0])).toContain("Watch run run_1 until it starts."); - expect(JSON.stringify(stored?.[0])).toContain("Investigate straight away"); - expect(result.messages.map((message) => message.id)).toEqual( - stored?.map((message) => message.id) - ); - } - ); - - postgresTest( - "leaves a repairable state when the confirmation never lands, and the retry repairs it", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-repair"); - await seedChat(seeded); - - // The crash state: the request record is written and the watch is live, but the - // process died before the confirmation was appended. - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - message: { id: "watch-request:wreq_1", role: "user", parts: [] } as never, - }); - const created = await create({ seeded, chatId: "chat_1" }); - expect(created.ok).toBe(true); - if (!created.ok || !created.watching) return; - - const retry = await submit({ seeded, chatId: "chat_1", clientRequestId: "wreq_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(created.watchId); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${created.watchId}`, - ]); - - // Still exactly one watch: the repair loaded it rather than creating another. - const active = await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" }); - expect(active).toHaveLength(1); - } - ); - - postgresTest( - "a retried submit duplicates neither record", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-retry"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - const second = await submit({ seeded, chatId: "chat_1" }); - - expect(first.ok && second.ok).toBe(true); - if (!first.ok || !second.ok) return; - expect(second.repaired).toBe(true); - expect(second.watchId).toBe(first.watchId); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a genuinely different request still conflicts", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-conflict"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok) return; - - // Same condition, so the same identity, but a different window: not a retry. - const longer = await submit({ - seeded, - chatId: "chat_1", - clientRequestId: "wreq_2", - draft: draftFor({ ...RUN_START, maxHours: 6 }), - }); - expect(longer).toMatchObject({ ok: false, code: "duplicate", existingId: first.watchId }); - - // Same spec, different consent: also not a retry. - const investigating = await submit({ - seeded, - chatId: "chat_1", - clientRequestId: "wreq_3", - draft: draftFor(RUN_START, { investigateOnAttention: true }), - }); - expect(investigating).toMatchObject({ ok: false, code: "duplicate" }); - - // The refused attempts are recorded under their own consent records, so the - // transcript never shows a request with no answer. - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - "watch-request:wreq_2", - "watch-confirmation:refused:wreq_2", - "watch-request:wreq_3", - "watch-confirmation:refused:wreq_3", - ]); - } - ); - - postgresTest( - "a fresh panel's retry reuses the chat the first attempt created", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-fresh"); - - const first = await submit({ seeded, clientRequestId: "wreq_fresh" }); - const second = await submit({ seeded, clientRequestId: "wreq_fresh" }); - - expect(first.ok && second.ok).toBe(true); - if (!first.ok || !second.ok) return; - expect(second.chatId).toBe(first.chatId); - - const stored = await storedMessages(seeded, first.chatId); - expect(stored).toHaveLength(2); - } - ); - - postgresTest( - "an answered condition records the request and a one-shot result, and never a watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-oneshot"); - await seedChat(seeded); - - const result = await submit({ - seeded, - chatId: "chat_1", - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(false); - expect(result.watchId).toBeNull(); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - "watch-confirmation:one-shot:wreq_1", - ]); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toHaveLength(0); - } - ); - - /** Every watch row for a chat, terminal ones included. `listActiveWatchesForChat` can't see those. */ - async function countWatchRows(prisma: PrismaClient, chatId: string) { - const rows = await prisma.$queryRawUnsafe>( - `select count(*)::bigint as count from trigger_dashboard_agent.watches where chat_id = $1`, - chatId - ); - return Number(rows[0]?.count ?? 0); - } - - postgresTest( - "a retry after the watch has already fired creates no second watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-fired"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok || !first.watchId) return; - - // The watch resolves and leaves the active set, so a duplicate check would find - // nothing. Only the ledger still knows this request already ran. - await transitionWatchCondition(ctx.agentDb, { - id: first.watchId, - resolution: "condition_met", - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(first.watchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a retry of an answered one-shot never becomes a watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-oneshot-retry"); - await seedChat(seeded); - - const first = await submit({ - seeded, - chatId: "chat_1", - checkDeps: { readRun: async () => runRow({ status: "EXECUTING", startedAt: new Date() }) }, - }); - expect(first.ok && first.watching === false).toBe(true); - - // The world moved on: the same condition would now be pending, so a re-evaluation - // would start a real watch. The recorded outcome is replayed instead. - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.watching).toBe(false); - expect(retry.watchId).toBeNull(); - expect(retry.repaired).toBe(true); - expect(await countWatchRows(prisma, "chat_1")).toBe(0); - - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - "watch-confirmation:one-shot:wreq_1", - ]); - } - ); - - postgresTest( - "the same request id carrying a different draft is a conflict", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-hash"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok) return; - - const changed = await submit({ - seeded, - chatId: "chat_1", - draft: draftFor({ ...RUN_START, maxHours: 6 }), - }); - expect(changed).toMatchObject({ ok: false, code: "request_conflict" }); - - // A conflict writes nothing at all: no watch, and no record under the request. - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - const stored = await storedMessages(seeded, "chat_1"); - expect(stored?.map((message) => message.id)).toEqual([ - "watch-request:wreq_1", - `watch-confirmation:${first.watchId}`, - ]); - } - ); - - postgresTest( - "a pending submission converges on the watch its first attempt created", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-converge"); - await seedChat(seeded); - - // The crash state the ledger exists for: the row is reserved, the watch is live - // under the reserved id, and the process died before the outcome was written. - let reservedWatchId = ""; - await expect( - submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - await createDashboardAgentWatch(createParams); - throw new Error("died after the watch was created"); - }, - }) - ).rejects.toThrow("died after the watch was created"); - - const pending = await getWatchSubmission(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - }); - expect(pending).toMatchObject({ state: "pending", watchId: reservedWatchId }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - // Reached the reserved row rather than creating another. - expect(retry.watchId).toBe(reservedWatchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const settled = await getWatchSubmission(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - }); - expect(settled).toMatchObject({ state: "created", watchId: reservedWatchId }); - } - ); - - postgresTest( - "converging on a watch that already fired confirms the outcome, not 'watching'", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-converge-fired"); - await seedChat(seeded); - - let reservedWatchId = ""; - await expect( - submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - await createDashboardAgentWatch(createParams); - throw new Error("died after the watch was created"); - }, - }) - ).rejects.toThrow("died after the watch was created"); - - // The watch ran and woke the chat before anyone retried the submit. - await transitionWatchCondition(ctx.agentDb, { - id: reservedWatchId, - resolution: "condition_met", - observedOutcome: { kind: "run_start", verified: true, status: "EXECUTING", started: true }, - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - // Still one row, still the same watch: adoption is not refused. - expect(retry.watchId).toBe(reservedWatchId); - expect(await countWatchRows(prisma, "chat_1")).toBe(1); - - const parts = retry.messages.at(-1)?.parts ?? []; - const block = (parts[0] as any).data.blocks[0]; - expect(block.outcome).toBe("already_true"); - expect(block.headline).not.toContain("Watching"); - expect(block.lifetime).toBeNull(); - } - ); - - postgresTest( - "a refusal that wins the race leaves no live watch behind", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-refused-race"); - await seedChat(seeded); - - // A concurrent attempt refuses this submission after the watch exists under the - // reserved id, so the ledger's winner keeps naming that id. - let reservedWatchId = ""; - const result = await submit({ - seeded, - chatId: "chat_1", - create: async (createParams) => { - reservedWatchId = createParams.watchId!; - const created = await createDashboardAgentWatch(createParams); - const refused = await recordWatchSubmissionOutcome(ctx.agentDb, { - chatId: "chat_1", - clientRequestId: "wreq_1", - state: "refused", - refusalCode: "internal", - refusalError: "That watch couldn't be started.", - }); - expect(refused).toMatchObject({ state: "refused", watchId: reservedWatchId }); - return created; - }, - }); - - // The user is told nothing is being watched, so nothing may be watching. - expect(result.ok).toBe(false); - const row = await getWatch(ctx.agentDb, { id: reservedWatchId }); - expect(row).toMatchObject({ status: "cancelled", cancelReason: "superseded" }); - } - ); - - postgresTest( - "the consent record never spends a message from the cap", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-quota"); - await seedChat(seeded); - - await submit({ seeded, chatId: "chat_1" }); - - expect( - await countUserMessages(ctx.agentDb, { - organizationId: seeded.organization.id, - userId: seeded.user.id, - }) - ).toBe(0); - } - ); - - postgresTest( - "a replay repeats the recorded email outcome and subscribes nobody", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-external-replay"); - await seedChat(seeded); - - const draft = draftFor(RUN_START, { notifyExternally: true }); - - // The first attempt asked for email and couldn't get it, so `unavailable` is what - // the transcript says and what the ledger records. - const first = await submit({ seeded, chatId: "chat_1", draft, subscribed: false }); - expect(first.ok).toBe(true); - if (!first.ok) return; - expect(JSON.stringify(first.messages)).toContain("I couldn't add email notifications"); - expect( - await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) - ).toMatchObject({ state: "created", externalNotificationStatus: "unavailable" }); - - const transcript = await storedMessages(seeded, "chat_1"); - - // The retry gets the real subscribe, which would succeed here. A replay that took the - // decision again would leave a channel row and an `enabled` answer the transcript โ€” - // append-once, so never rewritten โ€” contradicts for good. - let subscribeCalls = 0; - const retry = await submit({ - seeded, - chatId: "chat_1", - draft, - subscribe: async (subscribeParams) => { - subscribeCalls++; - return subscribeUserToWatchAlerts(subscribeParams); - }, - }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - expect(retry.watchId).toBe(first.watchId); - expect(subscribeCalls).toBe(0); - - expect(JSON.stringify(retry.messages)).toContain("I couldn't add email notifications"); - expect(JSON.stringify(retry.messages)).not.toContain("You'll get an email"); - expect( - await prisma.projectAlertChannel.count({ where: { projectId: seeded.project.id } }) - ).toBe(0); - expect( - await getWatchSubmission(ctx.agentDb, { chatId: "chat_1", clientRequestId: "wreq_1" }) - ).toMatchObject({ externalNotificationStatus: "unavailable" }); - - // The symptom: what the user is told after a refresh has to agree with the answer. - expect(await storedMessages(seeded, "chat_1")).toEqual(transcript); - } - ); - - postgresTest( - "a replay repeats the recorded 'Watching' confirmation after the watch has fired", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "submit-replay-fired"); - await seedChat(seeded); - - const first = await submit({ seeded, chatId: "chat_1" }); - expect(first.ok).toBe(true); - if (!first.ok || !first.watchId) return; - - await transitionWatchCondition(ctx.agentDb, { - id: first.watchId, - resolution: "condition_met", - }); - - const retry = await submit({ seeded, chatId: "chat_1" }); - - expect(retry.ok).toBe(true); - if (!retry.ok) return; - expect(retry.repaired).toBe(true); - - // The recorded outcome is replayed, never decided again: the append-once - // confirmation in the transcript says "Watching", so the answer has to as well. - const parts = retry.messages.at(-1)?.parts ?? []; - const block = (parts[0] as any).data.blocks[0]; - expect(block.outcome).toBe("watching"); - expect(block.headline).toContain("Watching"); - } - ); -}); - -describe("appendChatMessageOnce", () => { - postgresTest( - "appends in order without rewriting the transcript", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "append"); - await seedChat(seeded); - - const first = { id: "watch-card:watch_1", role: "assistant", parts: [] }; - const second = { id: "watch-card:watch_2", role: "assistant", parts: [] }; - - expect( - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - message: first, - }) - ).toBe(true); - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - message: second, - }); - - const messages = await getChatMessages(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - expect(messages).toEqual([first, second]); - } - ); - - postgresTest( - "appends nothing for a chat the caller doesn't own", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "append-owner"); - await seedChat(seeded); - - expect( - await appendChatMessageOnce(ctx.agentDb, { - chatId: "chat_1", - userId: "user_someone_else", - organizationId: seeded.organization.id, - message: { id: "watch-card:watch_1", role: "assistant", parts: [] }, - }) - ).toBe(false); - - const messages = await getChatMessages(ctx.agentDb, { - chatId: "chat_1", - userId: seeded.user.id, - organizationId: seeded.organization.id, - }); - expect(messages).toEqual([]); - } - ); -}); - -describe("run_failed creation", () => { - const RUN_FAILED: WatchSpec = { - kind: "run_failed", - runId: "run_1", - checkEveryMinutes: 1, - maxHours: 2, - note: "tell me if it fails", - }; - - postgresTest( - "watches a running run and dedups against the finished variant separately", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "runfailed"); - await seedChat(seeded); - - const failed = await create({ - seeded, - spec: RUN_FAILED, - checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, - }); - expect(failed.ok).toBe(true); - if (!failed.ok || !failed.watching) return; - expect(failed.identity).toBe("run_failed:run_1"); - - const finished = await create({ - seeded, - spec: { ...RUN_FAILED, kind: "run_finished" } as WatchSpec, - checkDeps: { readRun: async () => runRow({ status: "EXECUTING" }) }, - }); - expect(finished.ok).toBe(true); - if (!finished.ok || !finished.watching) return; - expect(finished.identity).toBe("run_finished:run_1"); - } - ); - - postgresTest( - "answers outright, with no watch row, once the run has succeeded", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "runfailed-done"); - await seedChat(seeded); - - const result = await create({ - seeded, - spec: RUN_FAILED, - checkDeps: { - readRun: async () => - runRow({ status: "COMPLETED_SUCCESSFULLY", completedAt: new Date() }), - }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(false); - if (result.watching) return; - expect(result.immediate.result).toBe("terminal_unsatisfied"); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); - } - ); -}); - -describe("the queue pack creation", () => { - const QUEUE = "task/my-task"; - - const BELOW: WatchSpec = { - kind: "queue_depth_below", - queue: QUEUE, - threshold: 100, - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me when it's back below 100", - }; - - const STALLED: WatchSpec = { - kind: "queue_stalled", - queue: QUEUE, - ticks: 3, - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me if it stops moving", - }; - - const AGE: WatchSpec = { - kind: "queue_oldest_age", - queue: QUEUE, - thresholdMinutes: 5, - checkEveryMinutes: 5, - maxHours: 2, - note: "tell me if runs wait longer than 5 minutes", - }; - - postgresTest( - "creates each kind with its own identity on the same queue", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queuepack"); - await seedChat(seeded); - - const busy = { - readQueueDepth: async () => ({ - depth: 780, - source: "live_queue" as const, - current: true, - }), - }; - - const below = await create({ seeded, spec: BELOW, checkDeps: busy }); - expect(below.ok && below.watching).toBe(true); - if (!below.ok || !below.watching) return; - expect(below.identity).toBe(`queue_depth_below:${QUEUE}:100`); - - const stalled = await create({ seeded, spec: STALLED, checkDeps: busy }); - expect(stalled.ok && stalled.watching).toBe(true); - if (!stalled.ok || !stalled.watching) return; - expect(stalled.identity).toBe(`queue_stalled:${QUEUE}`); - - const age = await create({ seeded, spec: AGE, checkDeps: busy }); - expect(age.ok && age.watching).toBe(true); - if (!age.ok || !age.watching) return; - expect(age.identity).toBe(`queue_oldest_age:${QUEUE}:5`); - - const drain = await create({ - seeded, - spec: { ...BELOW, kind: "backlog_drain" } as WatchSpec, - checkDeps: busy, - }); - expect(drain.ok).toBe(false); - if (drain.ok) return; - expect(drain.code).toBe("limit_reached"); - } - ); - - postgresTest( - "dedups the same SLA and allows a different one", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queueage"); - await seedChat(seeded); - - const first = await create({ seeded, spec: AGE }); - expect(first.ok && first.watching).toBe(true); - - const same = await create({ seeded, spec: AGE }); - expect(same.ok).toBe(false); - if (same.ok) return; - expect(same.code).toBe("duplicate"); - - const other = await create({ seeded, spec: { ...AGE, thresholdMinutes: 30 } as WatchSpec }); - expect(other.ok && other.watching).toBe(true); - if (!other.ok || !other.watching) return; - expect(other.identity).toBe(`queue_oldest_age:${QUEUE}:30`); - } - ); - - postgresTest( - "answers a back-below ask outright when the queue is already quiet", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queuebelow"); - await seedChat(seeded); - - const result = await create({ - seeded, - spec: BELOW, - checkDeps: { - readQueueDepth: async () => ({ depth: 4, source: "live_queue", current: true }), - }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.watching).toBe(false); - if (result.watching) return; - expect(result.immediate.result).toBe("satisfied"); - expect(await listActiveWatchesForChat(ctx.agentDb, { chatId: "chat_1" })).toEqual([]); - } - ); - - postgresTest( - "round-trips the stall state through the row's existing facts column", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "queuestall"); - await seedChat(seeded); - - const created = await create({ - seeded, - spec: STALLED, - checkDeps: { - readQueueDepth: async () => ({ depth: 42, source: "live_queue", current: true }), - }, - }); - expect(created.ok && created.watching).toBe(true); - if (!created.ok || !created.watching) return; - - const facts = { queue: QUEUE, depth: 42, notDecreasingStreak: 2, ticks: 3 }; - await recordWatchCheck(ctx.agentDb, { - id: created.watchId, - lastResult: { - result: "pending", - facts, - observed: { - kind: "queue_stalled", - verified: true, - depth: 42, - notDecreasingStreak: 2, - ticks: 3, - }, - final: false, - }, - }); - - const row = await getWatch(ctx.agentDb, { id: created.watchId }); - expect(previousCheckFacts(row?.lastResult)).toEqual(facts); - - await recordWatchCheck(ctx.agentDb, { - id: created.watchId, - lastResult: { checkFailed: true, detail: "clickhouse down", previous: facts }, - }); - const afterGap = await getWatch(ctx.agentDb, { id: created.watchId }); - expect(previousCheckFacts(afterGap?.lastResult)).toEqual(facts); - } - ); -}); - -const HEALTH: WatchSpec = { - kind: "health_recovery", - report: "health", - fromSeverity: "warn", - checkEveryMinutes: 5, - maxHours: 6, - note: "tell me when health recovers", -}; - -describe("the batch chain registry", () => { - postgresTest("arms one chain per group, and only one", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batcharm"); - const now = new Date(); - - const scheduled: Array<{ epoch: number; tick: number }> = []; - const arm = () => - armDashboardAgentWatchBatch({ - environmentId: seeded.environment.id, - cadenceMinutes: 5, - now, - deps: { - schedule: async (params) => - void scheduled.push({ epoch: params.epoch, tick: params.tick }), - }, - }); - - expect(await arm()).toEqual({ running: true }); - expect(scheduled).toEqual([{ epoch: 1, tick: 1 }]); - - expect(await arm()).toEqual({ running: true }); - expect(await arm()).toEqual({ running: true }); - expect(scheduled).toHaveLength(1); - }); - - postgresTest( - "a chain whose run died is re-armed on a fresh epoch, and the zombie claims nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchdead"); - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; - - const scheduled: Array<{ epoch: number; tick: number }> = []; - const arm = (now: Date) => - armDashboardAgentWatchBatch({ - ...group, - now, - deps: { - schedule: async (params) => - void scheduled.push({ epoch: params.epoch, tick: params.tick }), - }, - }); - - const armedAt = new Date(); - await arm(armedAt); - expect( - await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 1 }) - ).toMatchObject({ epoch: 1, generation: 1 }); - - await arm(new Date(armedAt.getTime() + 60_000)); - expect(scheduled).toHaveLength(1); - - await arm(new Date(armedAt.getTime() + watchBatchStaleMs(5) + 60_000)); - expect(scheduled).toEqual([ - { epoch: 1, tick: 1 }, - { epoch: 2, tick: 1 }, - ]); - - expect(await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 1, generation: 2 })).toBe( - null - ); - expect( - await claimWatchBatchTick(ctx.agentDb, { ...group, epoch: 2, generation: 1 }) - ).toMatchObject({ epoch: 2, generation: 1 }); - } - ); - - postgresTest( - "a chain that couldn't be triggered is not left marked as running", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchfail"); - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; - - expect( - await armDashboardAgentWatchBatch({ - ...group, - deps: { - schedule: async () => { - throw new Error("the trigger failed"); - }, - }, - }) - ).toEqual({ running: false }); - - const scheduled: number[] = []; - expect( - await armDashboardAgentWatchBatch({ - ...group, - deps: { schedule: async (params) => void scheduled.push(params.epoch) }, - }) - ).toEqual({ running: true }); - expect(scheduled).toEqual([2]); - } - ); - - postgresTest( - "the re-arm backstop finds groups with active watches and no chain", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchrearm"); - await seedChat(seeded); - const created = await create({ - seeded, - spec: HEALTH, - checkDeps: { readHealth: async () => null }, - }); - expect(created.ok).toBe(true); - - const groups = await listWatchBatchGroupsToArm(ctx.agentDb); - expect(groups).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); - - const armed: Array<{ environmentId: string; cadenceMinutes: number }> = []; - expect( - await rearmDashboardAgentWatchBatches({ - configured: () => true, - arm: async (params) => { - armed.push({ - environmentId: params.environmentId, - cadenceMinutes: params.cadenceMinutes, - }); - return { running: true }; - }, - }) - ).toEqual({ stale: 1, armed: 1, failed: 0 }); - expect(armed).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); - - // The staleness window is the group's own cadence: a five-minute group goes stale 17 minutes later. - await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - staleBefore: new Date(), - }); - expect(await listWatchBatchGroupsToArm(ctx.agentDb)).toEqual([]); - expect( - await listWatchBatchGroupsToArm(ctx.agentDb, { - now: new Date(Date.now() + watchBatchStaleMs(5) + 60_000), - }) - ).toEqual([{ environmentId: seeded.environment.id, cadenceMinutes: 5 }]); - } - ); - - postgresTest( - "groups are per environment and per cadence, never mixed", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchgroup"); - await seedChat(seeded, "chat_1"); - await seedChat(seeded, "chat_2"); - expect((await create({ seeded, chatId: "chat_1", spec: HEALTH })).ok).toBe(true); - expect((await create({ seeded, chatId: "chat_2", spec: RUN_START })).ok).toBe(true); - - const five = await listActiveWatchesForBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - }); - const one = await listActiveWatchesForBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 1, - }); - - expect(five.map((watch) => watch.chatId)).toEqual(["chat_1"]); - expect(one.map((watch) => watch.chatId)).toEqual(["chat_2"]); - expect( - (await listWatchBatchGroupsToArm(ctx.agentDb)).sort( - (a, b) => a.cadenceMinutes - b.cadenceMinutes - ) - ).toEqual([ - { environmentId: seeded.environment.id, cadenceMinutes: 1 }, - { environmentId: seeded.environment.id, cadenceMinutes: 5 }, - ]); - } - ); -}); - -describe("the batch check", () => { - async function healthGroup(seeded: Seeded, count = 3) { - const ids: string[] = []; - for (let index = 0; index < count; index++) { - const chatId = `chat_${index + 1}`; - await seedChat(seeded, chatId); - const created = await create({ - seeded, - chatId, - spec: HEALTH, - // `warn` keeps them all pending, so the group stays whole for the assertions below. - checkDeps: { readHealth: async () => ({ trustworthy: true, severity: "warn" }) }, - }); - if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); - ids.push(created.watchId); - } - return ids; - } - - async function otherUsersWatch(seeded: Seeded, prisma: PrismaClient) { - const user = await prisma.user.create({ - data: { - email: `other_${seeded.organization.slug}@example.com`, - authenticationMethod: "MAGIC_LINK", - }, - }); - await prisma.orgMember.create({ - data: { organizationId: seeded.organization.id, userId: user.id, role: "MEMBER" }, - }); - await createChat(ctx.agentDb, { - id: "chat_other", - organizationId: seeded.organization.id, - userId: user.id, - }); - const created = await createDashboardAgentWatch({ - environment: authenticated(seeded), - userId: user.id, - chatId: "chat_other", - spec: HEALTH, - deps: { - configured: () => true, - checkDeps: () => - fakeCheckDeps({ readHealth: async () => ({ trustworthy: true, severity: "warn" }) }), - scheduleTick: async () => {}, - }, - }); - if (!created.ok || !created.watching) throw new Error("the watch wasn't created"); - return { userId: user.id, watchId: created.watchId }; - } - - async function armChain(seeded: Seeded, cadenceMinutes = 5) { - const row = await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes, - staleBefore: new Date(), - }); - if (!row) throw new Error("the chain wasn't armed"); - return row; - } - - postgresTest( - "authorizes once and loads the shared report once for the whole group", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchcheck"); - const ids = await healthGroup(seeded); - const chain = await armChain(seeded); - - let healthReads = 0; - let authorizations = 0; - - const response = await runWatchBatchCheck( - { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain.epoch, - tick: 1, - }, - { - authorize: async () => { - authorizations++; - return { ok: true, environment: authenticated(seeded) }; - }, - checkDeps: () => - fakeCheckDeps({ - readHealth: async () => { - healthReads++; - return { trustworthy: true, severity: "warn" }; - }, - }), - } - ); - - expect(authorizations).toBe(1); - expect(healthReads).toBe(1); - - expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual([...ids].sort()); - expect(response.watches?.every((entry) => entry.result === "pending")).toBe(true); - expect(response.watches?.every((entry) => entry.tick === 1)).toBe(true); - expect(response.watches?.every((entry) => entry.token.length > 0)).toBe(true); - expect(response.continues).toBe(true); - expect(response.stale).toBeUndefined(); - - for (const id of ids) { - expect((await getWatch(ctx.agentDb, { id }))?.lastResult).toMatchObject({ - result: "pending", - final: false, - }); - } - } - ); - - postgresTest( - "authorizes each distinct user, so sharing readers never shares access", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchusers"); - await healthGroup(seeded, 2); - const other = await otherUsersWatch(seeded, prisma); - - const chain = await armChain(seeded); - const authorized: string[] = []; - - await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - authorize: async (watch) => { - authorized.push(watch.userId); - return { ok: true, environment: authenticated(seeded) }; - }, - checkDeps: () => fakeCheckDeps(), - } - ); - - expect(authorized.sort()).toEqual([other.userId, seeded.user.id].sort()); - } - ); - - postgresTest( - "cancels a watch whose user lost access, and still answers for its neighbours", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchrevoked"); - const ids = await healthGroup(seeded, 2); - const chain = await armChain(seeded); - - const response = await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - authorize: async () => ({ ok: false, reason: "access_revoked" }), - checkDeps: () => fakeCheckDeps(), - } - ); - - expect(response.watches?.every((entry) => entry.code === "access_revoked")).toBe(true); - for (const id of ids) { - expect(await getWatch(ctx.agentDb, { id })).toMatchObject({ - status: "cancelled", - cancelReason: "access_revoked", - deliveryStatus: "not_required", - }); - } - } - ); - - postgresTest( - "checks what is due, skips what isn't, and never skips a window boundary", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchdue"); - const [fresh, overdue, boundary] = await healthGroup(seeded, 3); - const chain = await armChain(seeded); - const now = new Date(); - - await recordWatchCheck(ctx.agentDb, { id: fresh!, lastCheckedAt: now }); - await recordWatchCheck(ctx.agentDb, { - id: overdue!, - lastCheckedAt: new Date(now.getTime() - 10 * 60_000), - }); - // `boundary`'s window closes before the next tick, so its final evaluation must still happen. - await recordWatchCheck(ctx.agentDb, { id: boundary!, lastCheckedAt: now }); - await prisma.$executeRawUnsafe( - `update trigger_dashboard_agent.watches set expires_at = now() + interval '1 minute' where id = $1`, - boundary - ); - - const response = await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - now: () => now, - authorize: async () => ({ ok: true, environment: authenticated(seeded) }), - checkDeps: () => fakeCheckDeps(), - } - ); - - expect(response.watches?.map((entry) => entry.watchId).sort()).toEqual( - [boundary!, overdue!].sort() - ); - expect(response.continues).toBe(true); - } - ); - - postgresTest( - "a stale tick claims nothing and checks nothing", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchstale"); - const ids = await healthGroup(seeded, 1); - const chain = await armChain(seeded); - - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch }; - expect((await runWatchBatchCheck({ ...group, tick: 1 })).stale).toBeUndefined(); - expect((await runWatchBatchCheck({ ...group, tick: 2 })).stale).toBeUndefined(); - - const late = await runWatchBatchCheck({ ...group, tick: 1 }); - expect(late).toEqual({ stale: true }); - - expect(await runWatchBatchCheck({ ...group, epoch: chain.epoch - 1, tick: 1 })).toEqual({ - stale: true, - }); - expect((await getWatch(ctx.agentDb, { id: ids[0]! }))?.status).toBe("active"); - } - ); - - postgresTest( - "stops the chain when the group's last watch is gone", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchempty"); - const ids = await healthGroup(seeded, 1); - const chain = await armChain(seeded); - await cancelWatch(ctx.agentDb, { id: ids[0]!, reason: "user" }); - - const response = await runWatchBatchCheck({ - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain.epoch, - tick: 1, - }); - - expect(response).toMatchObject({ watches: [], continues: false }); - - const rearmed = await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - // Deliberately in the past: only a stopped chain can be re-armed this way. - staleBefore: new Date(Date.now() - 60 * 60_000), - }); - expect(rearmed).toMatchObject({ epoch: chain.epoch + 1, status: "running" }); - } - ); - - postgresTest( - "hands the group's owed wakes back for redelivery", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchowed"); - const ids = await healthGroup(seeded, 2); - const chain = await armChain(seeded); - - await transitionWatchCondition(ctx.agentDb, { - id: ids[0]!, - resolution: "condition_met", - lastResult: { verified: true }, - }); - - const response = await runWatchBatchCheck({ - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain.epoch, - tick: 1, - }); - - const owed = response.watches?.filter((entry) => entry.deliverOnly === true) ?? []; - expect(owed.map((entry) => entry.watchId)).toEqual([ids[0]!]); - expect(owed[0]?.tick).toBe(0); - expect( - response.watches?.filter((entry) => !entry.deliverOnly).map((entry) => entry.watchId) - ).toEqual([ids[1]!]); - } - ); - - postgresTest( - "keeps the chain alive while a wake is still owed, even with nothing left to watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchowedlast"); - const ids = await healthGroup(seeded, 1); - const chain = await armChain(seeded); - const group = { environmentId: seeded.environment.id, cadenceMinutes: 5 }; - - await transitionWatchCondition(ctx.agentDb, { - id: ids[0]!, - resolution: "condition_met", - lastResult: { verified: true }, - }); - - const first = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 1 }); - expect(first.continues).toBe(true); - expect(first.watches?.map((entry) => entry.deliverOnly)).toEqual([true]); - - const claim = await claimWatchDelivery(ctx.agentDb, { - id: ids[0]!, - staleBefore: new Date(Date.now() - WATCH_DELIVERY_CLAIM_STALE_MS), - }); - await markWatchDelivered(ctx.agentDb, { id: ids[0]!, claimId: claim!.claimId }); - - const second = await runWatchBatchCheck({ ...group, epoch: chain.epoch, tick: 2 }); - expect(second).toMatchObject({ watches: [], continues: false }); - expect(await stopWatchBatch(ctx.agentDb, { ...group, epoch: chain.epoch })).toBe(null); - } - ); - - postgresTest( - "one watch that throws mid-evaluation costs only that watch", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchthrow"); - const mine = await healthGroup(seeded, 2); - const theirs = await otherUsersWatch(seeded, prisma); - const chain = await armChain(seeded); - - const response = await runWatchBatchCheck( - { environmentId: seeded.environment.id, cadenceMinutes: 5, epoch: chain.epoch, tick: 1 }, - { - authorize: async (watch) => { - if (watch.userId === theirs.userId) throw new Error("the authorization query failed"); - return { ok: true, environment: authenticated(seeded) }; - }, - checkDeps: () => fakeCheckDeps(), - concurrency: 1, - } - ); - - const byId = new Map(response.watches?.map((entry) => [entry.watchId, entry])); - expect(byId.get(theirs.watchId)).toMatchObject({ result: "unavailable" }); - expect((await getWatch(ctx.agentDb, { id: theirs.watchId }))?.status).toBe("active"); - for (const id of mine) { - expect(byId.get(id)).toMatchObject({ result: "pending" }); - } - } - ); -}); - -describe("the batch check endpoint's authorization", () => { - function batchRequest(body: unknown, token?: string) { - return new Request("https://app.trigger.dev/api/v1/dashboard-agent/watches/batch-check", { - method: "POST", - headers: { - ...(token ? { Authorization: `Bearer ${token}` } : {}), - "Content-Type": "application/json", - }, - body: JSON.stringify(body), - }); - } - - const batchToken = (environmentId: string, cadenceMinutes: number) => - signDashboardAgentWatchBatchToken(SESSION_SECRET, { - environmentId, - cadenceMinutes, - expiresAt: new Date(Date.now() + 60 * 60_000), - }); - - postgresTest("refuses a missing or bad token", async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const body = { environmentId: "env_1", cadenceMinutes: 5, epoch: 1, tick: 1 }; - - expect( - (await batchCheckAction({ request: batchRequest(body), params: {}, context: {} })).status - ).toBe(401); - const watchToken = await signDashboardAgentWatchToken(SESSION_SECRET, { - watchId: "watch_1", - expiresAt: new Date(Date.now() + 60 * 60_000), - }); - expect( - (await batchCheckAction({ request: batchRequest(body, watchToken), params: {}, context: {} })) - .status - ).toBe(401); - }); - - postgresTest( - "refuses a token minted for another group", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const token = await batchToken("env_1", 5); - - const wrongCadence = await batchCheckAction({ - request: batchRequest( - { environmentId: "env_1", cadenceMinutes: 15, epoch: 1, tick: 1 }, - token - ), - params: {}, - context: {}, - }); - expect(wrongCadence.status).toBe(403); - expect(await wrongCadence.json()).toMatchObject({ code: "group_mismatch" }); - - const wrongEnvironment = await batchCheckAction({ - request: batchRequest( - { environmentId: "env_2", cadenceMinutes: 5, epoch: 1, tick: 1 }, - token - ), - params: {}, - context: {}, - }); - expect(wrongEnvironment.status).toBe(403); - } - ); - - postgresTest( - "answers a group it does own, through the real registry", - async ({ prisma, postgresContainer }) => { - await boot(prisma, postgresContainer.getConnectionUri()); - const seeded = await seed(prisma, "batchroute"); - const chain = await armWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - staleBefore: new Date(), - }); - const token = await batchToken(seeded.environment.id, 5); - - const response = await batchCheckAction({ - request: batchRequest( - { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain!.epoch, - tick: 1, - }, - token - ), - params: {}, - context: {}, - }); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ watches: [], continues: false }); - expect( - await stopWatchBatch(ctx.agentDb, { - environmentId: seeded.environment.id, - cadenceMinutes: 5, - epoch: chain!.epoch, - }) - ).toBe(null); - } - ); -}); diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts new file mode 100644 index 000000000..b52e1fbb6 --- /dev/null +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.helpers.ts @@ -0,0 +1,117 @@ +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import type { + ExternalDeploymentCache, + ExternalDeploymentCacheEntry, +} from "~/services/externalDeploymentCache.server"; +import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server"; +import { + MockPayloadProcessor, + MockTraceEventConcern, + MockTriggerTaskValidator, +} from "./triggerTaskTestHelpers"; + +export class RecordingExternalDeploymentCache implements ExternalDeploymentCache { + readonly gets: Array<{ environmentId: string; externalId: string }> = []; + readonly writes: Array<{ + environmentId: string; + externalId: string; + entry: ExternalDeploymentCacheEntry; + }> = []; + readonly missing: Array<{ environmentId: string; externalId: string }> = []; + private readonly entries = new Map(); + + constructor( + entries: Array<{ + environmentId: string; + externalId: string; + entry: ExternalDeploymentCacheEntry; + }> = [] + ) { + for (const { environmentId, externalId, entry } of entries) { + this.entries.set(this.key(environmentId, externalId), entry); + } + } + + async get(environmentId: string, externalId: string) { + this.gets.push({ environmentId, externalId }); + + const entry = this.entries.get(this.key(environmentId, externalId)); + + if (entry) { + return { outcome: "deployed" as const, entry }; + } + + return this.missing.some( + (missing) => missing.environmentId === environmentId && missing.externalId === externalId + ) + ? { outcome: "missing" as const } + : null; + } + + async setIfNewer(environmentId: string, externalId: string, entry: ExternalDeploymentCacheEntry) { + this.writes.push({ environmentId, externalId, entry }); + this.entries.set(this.key(environmentId, externalId), entry); + } + + async setMissing(environmentId: string, externalId: string) { + this.missing.push({ environmentId, externalId }); + } + + private key(environmentId: string, externalId: string) { + return JSON.stringify([environmentId, externalId]); + } +} + +export function createEngine(prisma: PrismaClient, redisOptions: RedisOptions) { + return new RunEngine({ + prisma, + worker: { redis: redisOptions, disabled: true }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + ttlSystem: { disabled: true }, + }, + batchQueue: { redis: redisOptions, consumerEnabled: false }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { name: "small-1x", cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +export function createService( + prisma: PrismaClient, + engine: RunEngine, + externalDeploymentCache: ExternalDeploymentCache +) { + return new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()), + validator: new MockTriggerTaskValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024, + externalDeploymentCache, + }); +} + +export async function nameDeploymentWithExternalId( + prisma: PrismaClient, + workerId: string, + externalId: string +) { + await prisma.workerDeployment.update({ where: { workerId }, data: { externalId } }); +} diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts new file mode 100644 index 000000000..89026011f --- /dev/null +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, onTestFinished, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, +})); + +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { NoopExternalDeploymentCache } from "~/services/externalDeploymentCache.server"; +import { + createEngine, + createService, + RecordingExternalDeploymentCache, +} from "./triggerTask.externalDeploymentId.helpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +describe("triggerTask external deployment id", () => { + containerTest( + "parks a run whose id nothing holds, recording the id in annotations", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "parked-task"; + + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const service = createService(prisma, engine, new NoopExternalDeploymentCache()); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING_VERSION"); + expect(run.statusReason).toBe("EXTERNAL_DEPLOYMENT_PENDING"); + expect(run.lockedToVersionId).toBeNull(); + expect((run.annotations as Record).externalDeploymentId).toBe( + "commit-unknown" + ); + } + ); + + containerTest( + "never parks in development, where no deployment can ever hold the id", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "DEVELOPMENT"); + const taskIdentifier = "dev-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const cache = new RecordingExternalDeploymentCache(); + const service = createService(prisma, engine, cache); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING"); + expect(run.statusReason).toBeNull(); + expect(cache.gets).toEqual([]); + expect((run.annotations as Record).externalDeploymentId).toBe( + "commit-unknown" + ); + } + ); + + containerTest( + "parks a run whose id is held only by an in-flight deployment", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "inflight-task"; + + const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); + + await prisma.workerDeployment.update({ + where: { workerId: worker.worker.id }, + data: { externalId: "commit-building", status: "BUILDING" }, + }); + + const cache = new RecordingExternalDeploymentCache(); + const service = createService(prisma, engine, cache); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-building" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING_VERSION"); + expect(run.lockedToVersionId).toBeNull(); + + expect(cache.writes).toEqual([]); + } + ); +}); diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts new file mode 100644 index 000000000..b74b8c8f8 --- /dev/null +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, onTestFinished, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, +})); + +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { NoopExternalDeploymentCache } from "~/services/externalDeploymentCache.server"; +import { + createEngine, + createService, + RecordingExternalDeploymentCache, +} from "./triggerTask.externalDeploymentId.helpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +describe("triggerTask external deployment id", () => { + containerTest( + "trusts a cache hit without querying Postgres", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "cached-pin-task"; + + const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); + + const cache = new RecordingExternalDeploymentCache([ + { + environmentId: environment.id, + externalId: "commit-cached", + entry: { + workerId: worker.worker.id, + version: worker.worker.version, + sdkVersion: "", + cliVersion: "", + }, + }, + ]); + + const service = createService(prisma, engine, cache); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-cached" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING"); + expect(run.lockedToVersionId).toBe(worker.worker.id); + expect(cache.gets).toEqual([{ environmentId: environment.id, externalId: "commit-cached" }]); + expect(cache.writes).toEqual([]); + } + ); + + containerTest( + "resolves to the highest version when several deployed deployments hold the id", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "forced-task"; + + const older = await setupBackgroundWorker(engine, environment, taskIdentifier); + await prisma.backgroundWorker.update({ + where: { id: older.worker.id }, + data: { version: "20260807.9" }, + }); + await prisma.workerDeployment.update({ + where: { workerId: older.worker.id }, + data: { + externalId: "commit-forced", + version: "20260807.9", + shortCode: "short_code_20260807.9", + }, + }); + + const newer = await setupBackgroundWorker(engine, environment, taskIdentifier); + await prisma.backgroundWorker.update({ + where: { id: newer.worker.id }, + data: { version: "20260807.10" }, + }); + await prisma.workerDeployment.update({ + where: { workerId: newer.worker.id }, + data: { + externalId: "commit-forced", + version: "20260807.10", + shortCode: "short_code_20260807.10", + }, + }); + + const service = createService(prisma, engine, new NoopExternalDeploymentCache()); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-forced" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.lockedToVersionId).toBe(newer.worker.id); + expect(run.taskVersion).toBe("20260807.10"); + } + ); + + containerTest( + "an id is environment-scoped, so a deployment in another environment never resolves it", + async ({ prisma, redisOptions }) => { + const engine = createEngine(prisma, redisOptions); + onTestFinished(() => engine.quit()); + + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "scoped-task"; + + const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); + + const otherEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging-scoped", + type: "STAGING", + projectId: environment.project.id, + organizationId: environment.organization.id, + apiKey: "tr_stg_scoped", + pkApiKey: "pk_stg_scoped", + shortcode: "stg-scoped", + }, + }); + + await prisma.workerDeployment.create({ + data: { + friendlyId: "deployment_elsewhere", + contentHash: "hash", + shortCode: "sc_elsewhere", + version: worker.worker.version, + status: "DEPLOYED", + externalId: "commit-elsewhere", + projectId: environment.project.id, + environmentId: otherEnvironment.id, + }, + }); + + const cache = new RecordingExternalDeploymentCache([ + { + environmentId: otherEnvironment.id, + externalId: "commit-elsewhere", + entry: { + workerId: worker.worker.id, + version: worker.worker.version, + sdkVersion: "", + cliVersion: "", + }, + }, + ]); + const service = createService(prisma, engine, cache); + + const result = await service.call({ + taskId: taskIdentifier, + environment, + body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-elsewhere" } }, + }); + + assertNonNullable(result); + + const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); + + expect(run.status).toBe("PENDING_VERSION"); + expect(run.lockedToVersionId).toBeNull(); + expect(cache.gets).toEqual([ + { environmentId: environment.id, externalId: "commit-elsewhere" }, + ]); + expect(cache.missing).toEqual([ + { environmentId: environment.id, externalId: "commit-elsewhere" }, + ]); + } + ); +}); diff --git a/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts b/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts index d6dba9ccd..3fc33c68c 100644 --- a/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts +++ b/apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts @@ -17,110 +17,17 @@ vi.mock("~/services/platform.v3.server", async (importOriginal) => { }; }); -import { RunEngine } from "@internal/run-engine"; import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; import { assertNonNullable, containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; -import type { PrismaClient } from "@trigger.dev/database"; -import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; -import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; import { - type ExternalDeploymentCache, - type ExternalDeploymentCacheEntry, - NoopExternalDeploymentCache, -} from "~/services/externalDeploymentCache.server"; -import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server"; -import { - MockPayloadProcessor, - MockTraceEventConcern, - MockTriggerTaskValidator, -} from "./triggerTaskTestHelpers"; + createEngine, + createService, + nameDeploymentWithExternalId, + RecordingExternalDeploymentCache, +} from "./triggerTask.externalDeploymentId.helpers"; vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); -class RecordingExternalDeploymentCache implements ExternalDeploymentCache { - readonly gets: Array<{ environmentId: string; externalId: string }> = []; - readonly writes: Array<{ externalId: string; entry: ExternalDeploymentCacheEntry }> = []; - - constructor(private readonly entries = new Map()) {} - - readonly missing: string[] = []; - - async get(environmentId: string, externalId: string) { - this.gets.push({ environmentId, externalId }); - - const entry = this.entries.get(externalId); - - if (entry) { - return { outcome: "deployed" as const, entry }; - } - - return this.missing.includes(externalId) ? { outcome: "missing" as const } : null; - } - - async setIfNewer( - _environmentId: string, - externalId: string, - entry: ExternalDeploymentCacheEntry - ) { - this.writes.push({ externalId, entry }); - this.entries.set(externalId, entry); - } - - async setMissing(_environmentId: string, externalId: string) { - this.missing.push(externalId); - } -} - -function createEngine(prisma: PrismaClient, redisOptions: unknown) { - const engine = new RunEngine({ - prisma, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - worker: { redis: redisOptions as any, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - queue: { redis: redisOptions as any }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - runLock: { redis: redisOptions as any }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { name: "small-1x", cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, - }, - baseCostInCents: 0.0005, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); - - return engine; -} - -function createService( - prisma: PrismaClient, - engine: RunEngine, - externalDeploymentCache: ExternalDeploymentCache -) { - return new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()), - validator: new MockTriggerTaskValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024, - externalDeploymentCache, - }); -} - -async function nameDeploymentWithExternalId( - prisma: PrismaClient, - workerId: string, - externalId: string -) { - await prisma.workerDeployment.update({ where: { workerId }, data: { externalId } }); -} - describe("triggerTask external deployment id", () => { containerTest( "pins the run to the deployment holding the id, not to whatever is current", @@ -230,255 +137,4 @@ describe("triggerTask external deployment id", () => { expect(cache.gets).toEqual([]); } ); - - containerTest( - "parks a run whose id nothing holds, recording the id in annotations", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "parked-task"; - - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const service = createService(prisma, engine, new NoopExternalDeploymentCache()); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING_VERSION"); - expect(run.statusReason).toBe("EXTERNAL_DEPLOYMENT_PENDING"); - expect(run.lockedToVersionId).toBeNull(); - expect((run.annotations as Record).externalDeploymentId).toBe( - "commit-unknown" - ); - } - ); - - containerTest( - "never parks in development, where no deployment can ever hold the id", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "DEVELOPMENT"); - const taskIdentifier = "dev-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const cache = new RecordingExternalDeploymentCache(); - const service = createService(prisma, engine, cache); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-unknown" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING"); - expect(run.statusReason).toBeNull(); - expect(cache.gets).toEqual([]); - expect((run.annotations as Record).externalDeploymentId).toBe( - "commit-unknown" - ); - } - ); - - containerTest( - "parks a run whose id is held only by an in-flight deployment", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "inflight-task"; - - const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); - - await prisma.workerDeployment.update({ - where: { workerId: worker.worker.id }, - data: { externalId: "commit-building", status: "BUILDING" }, - }); - - const cache = new RecordingExternalDeploymentCache(); - const service = createService(prisma, engine, cache); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-building" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING_VERSION"); - expect(run.lockedToVersionId).toBeNull(); - - expect(cache.writes).toEqual([]); - } - ); - - containerTest( - "trusts a cache hit without querying Postgres", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "cached-pin-task"; - - const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); - - const cache = new RecordingExternalDeploymentCache( - new Map([ - [ - "commit-cached", - { - workerId: worker.worker.id, - version: worker.worker.version, - sdkVersion: "", - cliVersion: "", - }, - ], - ]) - ); - - const service = createService(prisma, engine, cache); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-cached" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING"); - expect(run.lockedToVersionId).toBe(worker.worker.id); - expect(cache.gets).toEqual([{ environmentId: environment.id, externalId: "commit-cached" }]); - expect(cache.writes).toEqual([]); - } - ); - - containerTest( - "resolves to the highest version when several deployed deployments hold the id", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "forced-task"; - - const older = await setupBackgroundWorker(engine, environment, taskIdentifier); - await prisma.backgroundWorker.update({ - where: { id: older.worker.id }, - data: { version: "20260807.9" }, - }); - await prisma.workerDeployment.update({ - where: { workerId: older.worker.id }, - data: { - externalId: "commit-forced", - version: "20260807.9", - shortCode: "short_code_20260807.9", - }, - }); - - const newer = await setupBackgroundWorker(engine, environment, taskIdentifier); - await prisma.backgroundWorker.update({ - where: { id: newer.worker.id }, - data: { version: "20260807.10" }, - }); - await prisma.workerDeployment.update({ - where: { workerId: newer.worker.id }, - data: { - externalId: "commit-forced", - version: "20260807.10", - shortCode: "short_code_20260807.10", - }, - }); - - const service = createService(prisma, engine, new NoopExternalDeploymentCache()); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-forced" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.lockedToVersionId).toBe(newer.worker.id); - expect(run.taskVersion).toBe("20260807.10"); - } - ); - - containerTest( - "an id is environment-scoped, so a deployment in another environment never resolves it", - async ({ prisma, redisOptions }) => { - const engine = createEngine(prisma, redisOptions); - onTestFinished(() => engine.quit()); - - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "scoped-task"; - - const worker = await setupBackgroundWorker(engine, environment, taskIdentifier); - - const otherEnvironment = await prisma.runtimeEnvironment.create({ - data: { - slug: "staging-scoped", - type: "STAGING", - projectId: environment.project.id, - organizationId: environment.organization.id, - apiKey: "tr_stg_scoped", - pkApiKey: "pk_stg_scoped", - shortcode: "stg-scoped", - }, - }); - - await prisma.workerDeployment.create({ - data: { - friendlyId: "deployment_elsewhere", - contentHash: "hash", - shortCode: "sc_elsewhere", - version: worker.worker.version, - status: "DEPLOYED", - externalId: "commit-elsewhere", - projectId: environment.project.id, - environmentId: otherEnvironment.id, - }, - }); - - const service = createService(prisma, engine, new NoopExternalDeploymentCache()); - - const result = await service.call({ - taskId: taskIdentifier, - environment, - body: { payload: { test: "x" }, options: { externalDeploymentId: "commit-elsewhere" } }, - }); - - assertNonNullable(result); - - const run = await prisma.taskRun.findFirstOrThrow({ where: { id: result.run.id } }); - - expect(run.status).toBe("PENDING_VERSION"); - expect(run.lockedToVersionId).toBeNull(); - } - ); }); diff --git a/apps/webapp/test/envConcurrencyLimitPause.server.test.ts b/apps/webapp/test/envConcurrencyLimitPause.server.test.ts index 7bb67c813..ea78de33b 100644 --- a/apps/webapp/test/envConcurrencyLimitPause.server.test.ts +++ b/apps/webapp/test/envConcurrencyLimitPause.server.test.ts @@ -1,15 +1,14 @@ -import { RunEngine } from "@internal/run-engine"; import { containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; import type { PrismaClient } from "@trigger.dev/database"; import type { RedisOptions } from "ioredis"; import { describe, expect, onTestFinished, vi } from "vitest"; -import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { - createRuntimeEnvironment, - createTestOrgProjectWithMember, - uniqueId, -} from "./fixtures/environmentVariablesFixtures"; + authEnv, + createEnvConcurrencyLimitPauseTestEngine, + type EnvConcurrencyLimitPauseTestEngine, + loadEnvConcurrencyLimitPauseServices, + seedProductionEnv, +} from "./helpers/envConcurrencyLimitPauseTestHelpers"; vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); @@ -18,100 +17,33 @@ vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); // real RunQueue state, so put a real RunEngine - built on the test's own Redis container - back // behind the singleton. No test here uses the no-op default. const { engineHolder } = vi.hoisted(() => ({ - engineHolder: { current: undefined as any }, + engineHolder: { current: undefined as EnvConcurrencyLimitPauseTestEngine | undefined }, })); vi.mock("~/v3/runEngine.server", () => ({ - engine: new Proxy({} as Record, { - get: (_target, prop) => engineHolder.current?.[prop as string], + engine: new Proxy({} as Record, { + get: (_target, prop) => + engineHolder.current ? Reflect.get(engineHolder.current, prop) : undefined, }), })); function useEngine(prisma: PrismaClient, redisOptions: RedisOptions) { - const engine = new RunEngine({ - prisma, - worker: { redis: redisOptions, disabled: true }, - queue: { redis: redisOptions, masterQueueConsumersDisabled: true }, - runLock: { redis: redisOptions }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, - }, - baseCostInCents: 0.0001, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); - + const engine = createEnvConcurrencyLimitPauseTestEngine(prisma, redisOptions); engineHolder.current = engine; onTestFinished(async () => { engineHolder.current = undefined; await engine.quit(); }); - return engine; } -// The import chain reaches module-level singletons that throw at load time when -// REDIS_HOST/REDIS_PORT are unset (autoIncrementCounter via triggerTaskV1), so the env must point -// at the redis container BEFORE the modules are imported. Hence dynamic imports; vitest runs each -// file in its own fork, so the env mutation cannot leak into other suites. -async function loadServices(redisOptions: RedisOptions) { - process.env.REDIS_HOST = redisOptions.host; - process.env.REDIS_PORT = String(redisOptions.port); - process.env.REDIS_TLS_DISABLED = "true"; - const [{ updateEnvConcurrencyLimits }, { PauseEnvironmentService }, runtimeEnvironment] = - await Promise.all([ - import("~/v3/runQueue.server"), - import("~/v3/services/pauseEnvironment.server"), - import("~/models/runtimeEnvironment.server"), - ]); - return { - updateEnvConcurrencyLimits, - PauseEnvironmentService, - authIncludeBase: runtimeEnvironment.authIncludeBase, - toAuthenticated: runtimeEnvironment.toAuthenticated, - }; -} - -type Loaded = Awaited>; - -async function authEnv( - loaded: Loaded, - prisma: PrismaClient, - environmentId: string -): Promise { - const row = await prisma.runtimeEnvironment.findFirstOrThrow({ - where: { id: environmentId }, - include: loaded.authIncludeBase, - }); - return loaded.toAuthenticated(row); -} - -async function seedProductionEnv(prisma: PrismaClient, maximumConcurrencyLimit: number) { - const { organization, project } = await createTestOrgProjectWithMember(prisma); - const environment = await createRuntimeEnvironment(prisma, { - projectId: project.id, - organizationId: organization.id, - type: "PRODUCTION", - slug: uniqueId("prod"), - }); - - await prisma.runtimeEnvironment.update({ - where: { id: environment.id }, - data: { maximumConcurrencyLimit }, - }); - - return { organization, project, environment }; -} - // An unset RunQueue limit reads back as the engine default (10), so neither the 0 nor the 17 // assertions below can pass just because a push never happened. -describe("updateEnvConcurrencyLimits", () => { +describe("updateEnvConcurrencyLimits with stale environments", () => { containerTest( "clamps to 0 when the environment is paused, even though the caller's copy says otherwise", async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); const engine = useEngine(prisma, redisOptions); const { environment } = await seedProductionEnv(prisma, 17); @@ -132,25 +64,10 @@ describe("updateEnvConcurrencyLimits", () => { } ); - containerTest( - "pushes the real limit for a running environment", - async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); - const engine = useEngine(prisma, redisOptions); - - const { environment } = await seedProductionEnv(prisma, 17); - const env = await authEnv(loaded, prisma, environment.id); - - await loaded.updateEnvConcurrencyLimits(env, undefined, prisma); - - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); - } - ); - containerTest( "restores the real limit when the environment was resumed while the request was in flight", async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); const engine = useEngine(prisma, redisOptions); const { environment } = await seedProductionEnv(prisma, 17); @@ -174,44 +91,4 @@ describe("updateEnvConcurrencyLimits", () => { expect(await engine.runQueue.getEnvConcurrencyLimit(whilePaused)).toBe(17); } ); - - containerTest( - "an explicit limit wins over the stored pause state", - async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); - const engine = useEngine(prisma, redisOptions); - - const { environment } = await seedProductionEnv(prisma, 17); - await prisma.runtimeEnvironment.update({ - where: { id: environment.id }, - data: { paused: true }, - }); - const env = await authEnv(loaded, prisma, environment.id); - - // How billing-limit converge restores a limit as it unpauses: the caller decides, no read. - await loaded.updateEnvConcurrencyLimits(env, 9, prisma); - - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(9); - } - ); - - containerTest( - "a pause writes 0 and a resume restores the limit", - async ({ prisma, redisOptions }) => { - const loaded = await loadServices(redisOptions); - const engine = useEngine(prisma, redisOptions); - - const { environment } = await seedProductionEnv(prisma, 17); - const service = new loaded.PauseEnvironmentService(prisma); - const env = await authEnv(loaded, prisma, environment.id); - - expect(await service.call(env, "paused")).toEqual({ success: true, state: "paused" }); - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(0); - - // The service holds an environment read before its own resume update, so `env.paused` is - // stale here too. - expect(await service.call(env, "resumed")).toEqual({ success: true, state: "resumed" }); - expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); - } - ); }); diff --git a/apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts b/apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts new file mode 100644 index 000000000..0662d49fc --- /dev/null +++ b/apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts @@ -0,0 +1,71 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import { describe, expect, onTestFinished, vi } from "vitest"; +import { + authEnv, + createEnvConcurrencyLimitPauseTestEngine, + type EnvConcurrencyLimitPauseTestEngine, + loadEnvConcurrencyLimitPauseServices, + seedProductionEnv, +} from "./helpers/envConcurrencyLimitPauseTestHelpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const { engineHolder } = vi.hoisted(() => ({ + engineHolder: { current: undefined as EnvConcurrencyLimitPauseTestEngine | undefined }, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: new Proxy({} as Record, { + get: (_target, prop) => + engineHolder.current ? Reflect.get(engineHolder.current, prop) : undefined, + }), +})); + +function useEngine(prisma: PrismaClient, redisOptions: RedisOptions) { + const engine = createEnvConcurrencyLimitPauseTestEngine(prisma, redisOptions); + engineHolder.current = engine; + onTestFinished(async () => { + engineHolder.current = undefined; + await engine.quit(); + }); + return engine; +} + +describe("updateEnvConcurrencyLimits directly", () => { + containerTest( + "pushes the real limit for a running environment", + async ({ prisma, redisOptions }) => { + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); + const engine = useEngine(prisma, redisOptions); + + const { environment } = await seedProductionEnv(prisma, 17); + const env = await authEnv(loaded, prisma, environment.id); + + await loaded.updateEnvConcurrencyLimits(env, undefined, prisma); + + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); + } + ); + + containerTest( + "an explicit limit wins over the stored pause state", + async ({ prisma, redisOptions }) => { + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); + const engine = useEngine(prisma, redisOptions); + + const { environment } = await seedProductionEnv(prisma, 17); + await prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { paused: true }, + }); + const env = await authEnv(loaded, prisma, environment.id); + + // How billing-limit converge restores a limit as it unpauses: the caller decides, no read. + await loaded.updateEnvConcurrencyLimits(env, 9, prisma); + + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(9); + } + ); +}); diff --git a/apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts b/apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts new file mode 100644 index 000000000..eb4570757 --- /dev/null +++ b/apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts @@ -0,0 +1,56 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import { describe, expect, onTestFinished, vi } from "vitest"; +import { + authEnv, + createEnvConcurrencyLimitPauseTestEngine, + type EnvConcurrencyLimitPauseTestEngine, + loadEnvConcurrencyLimitPauseServices, + seedProductionEnv, +} from "./helpers/envConcurrencyLimitPauseTestHelpers"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const { engineHolder } = vi.hoisted(() => ({ + engineHolder: { current: undefined as EnvConcurrencyLimitPauseTestEngine | undefined }, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: new Proxy({} as Record, { + get: (_target, prop) => + engineHolder.current ? Reflect.get(engineHolder.current, prop) : undefined, + }), +})); + +function useEngine(prisma: PrismaClient, redisOptions: RedisOptions) { + const engine = createEnvConcurrencyLimitPauseTestEngine(prisma, redisOptions); + engineHolder.current = engine; + onTestFinished(async () => { + engineHolder.current = undefined; + await engine.quit(); + }); + return engine; +} + +describe("PauseEnvironmentService", () => { + containerTest( + "a pause writes 0 and a resume restores the limit", + async ({ prisma, redisOptions }) => { + const loaded = await loadEnvConcurrencyLimitPauseServices(redisOptions); + const engine = useEngine(prisma, redisOptions); + + const { environment } = await seedProductionEnv(prisma, 17); + const service = new loaded.PauseEnvironmentService(prisma); + const env = await authEnv(loaded, prisma, environment.id); + + expect(await service.call(env, "paused")).toEqual({ success: true, state: "paused" }); + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(0); + + // The service holds an environment read before its own resume update, so `env.paused` is + // stale here too. + expect(await service.call(env, "resumed")).toEqual({ success: true, state: "resumed" }); + expect(await engine.runQueue.getEnvConcurrencyLimit(env)).toBe(17); + } + ); +}); diff --git a/apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts b/apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts new file mode 100644 index 000000000..e97b569ec --- /dev/null +++ b/apps/webapp/test/helpers/apiRunListPresenterTestHelpers.ts @@ -0,0 +1,178 @@ +import type { ClickHouse, TaskRunV2 } from "@internal/clickhouse"; +import type { PrismaClient, TaskRun, TaskRunStatus } from "@trigger.dev/database"; +import { z } from "zod"; + +export type SeedContext = { + organizationId: string; + projectId: string; + environmentId: string; + environmentSlug: string; +}; + +/** Creates the org/project/environment parents needed by TaskRun foreign keys. */ +export async function seedParents( + prisma: PrismaClient, + slug: string, + envSlug = `env-${slug}` +): Promise { + const organization = await prisma.organization.create({ + data: { title: `org-${slug}`, slug: `org-${slug}` }, + }); + const project = await prisma.project.create({ + data: { + name: `proj-${slug}`, + slug: `proj-${slug}`, + organizationId: organization.id, + externalRef: `proj-${slug}`, + }, + }); + const runtimeEnvironment = await prisma.runtimeEnvironment.create({ + data: { + slug: envSlug, + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slug}`, + pkApiKey: `pk_dev_${slug}`, + shortcode: `sc-${slug}`, + }, + }); + + return { + organizationId: organization.id, + projectId: project.id, + environmentId: runtimeEnvironment.id, + environmentSlug: runtimeEnvironment.slug, + }; +} + +/** Adds another control-plane environment to an existing project. */ +export async function addEnvironment( + prisma: PrismaClient, + ctx: SeedContext, + slug: string, + envSlug: string +): Promise { + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: envSlug, + type: "STAGING", + projectId: ctx.projectId, + organizationId: ctx.organizationId, + apiKey: `tr_${envSlug}_${slug}`, + pkApiKey: `pk_${envSlug}_${slug}`, + shortcode: `sc-${envSlug}-${slug}`, + }, + }); + + return environment.id; +} + +/** Mirrors the parents onto another database with the same IDs. */ +export async function mirrorParents( + prisma: PrismaClient, + ctx: SeedContext, + slug: string +): Promise { + await prisma.organization.create({ + data: { id: ctx.organizationId, title: `org-${slug}`, slug: `org-${slug}` }, + }); + await prisma.project.create({ + data: { + id: ctx.projectId, + name: `proj-${slug}`, + slug: `proj-${slug}`, + organizationId: ctx.organizationId, + externalRef: `proj-${slug}`, + }, + }); + await prisma.runtimeEnvironment.create({ + data: { + id: ctx.environmentId, + slug: ctx.environmentSlug, + type: "DEVELOPMENT", + projectId: ctx.projectId, + organizationId: ctx.organizationId, + apiKey: `tr_dev_${slug}_b`, + pkApiKey: `pk_dev_${slug}_b`, + shortcode: `sc-${slug}-b`, + }, + }); +} + +export async function createRun( + prisma: PrismaClient, + ctx: SeedContext, + run: { + friendlyId: string; + taskIdentifier?: string; + status?: TaskRunStatus; + runtimeEnvironmentId?: string; + } +): Promise { + return prisma.taskRun.create({ + data: { + friendlyId: run.friendlyId, + taskIdentifier: run.taskIdentifier ?? "my-task", + status: run.status ?? "PENDING", + payload: JSON.stringify({ foo: run.friendlyId }), + traceId: run.friendlyId, + spanId: run.friendlyId, + queue: "test", + runTags: [], + runtimeEnvironmentId: run.runtimeEnvironmentId ?? ctx.environmentId, + projectId: ctx.projectId, + organizationId: ctx.organizationId, + environmentType: "DEVELOPMENT", + engine: "V2", + }, + }); +} + +/** Inserts the ClickHouse list-index rows synchronously, without logical replication. */ +export async function insertTaskRunV2Rows(clickhouse: ClickHouse, runs: TaskRun[]): Promise { + const insert = clickhouse.writer.insert({ + name: "insertApiRunListPresenterTaskRuns", + table: "trigger_dev.task_runs_v2", + schema: z.any(), + settings: { async_insert: 0, enable_json_type: 1, type_json_skip_duplicated_paths: 1 }, + }); + + const rows: TaskRunV2[] = runs.map((run) => ({ + environment_id: run.runtimeEnvironmentId, + organization_id: run.organizationId ?? "", + project_id: run.projectId, + run_id: run.id, + friendly_id: run.friendlyId, + updated_at: run.updatedAt.getTime(), + created_at: run.createdAt.getTime(), + status: run.status, + environment_type: run.environmentType ?? "DEVELOPMENT", + attempt: run.attemptNumber ?? 1, + engine: run.engine, + task_identifier: run.taskIdentifier, + queue: run.queue, + schedule_id: "", + batch_id: "", + task_version: run.taskVersion ?? "", + sdk_version: run.sdkVersion ?? "", + cli_version: run.cliVersion ?? "", + machine_preset: run.machinePreset ?? "", + root_run_id: "", + parent_run_id: "", + span_id: run.spanId, + trace_id: run.traceId, + idempotency_key: run.idempotencyKey ?? "", + expiration_ttl: run.ttl ?? "", + tags: run.runTags, + worker_queue: run.workerQueue, + region: run.region ?? "", + _version: String(run.updatedAt.getTime()), + _is_deleted: 0, + })); + + const [error] = await insert(rows); + if (error) { + throw error; + } +} diff --git a/apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts b/apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts new file mode 100644 index 000000000..4562860bc --- /dev/null +++ b/apps/webapp/test/helpers/dashboardAgentWatchesTestHelpers.ts @@ -0,0 +1,219 @@ +import { + createChat, + createDashboardAgentDb, + getChatMessages, + type DashboardAgentDb, + type DashboardAgentDbClient, +} from "@internal/dashboard-agent-db"; +import { applyDashboardAgentMigrations } from "@internal/dashboard-agent-db/testing"; +import type { WatchDraft, WatchSpec } from "@internal/dashboard-agent-contracts"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { WatchCheckDeps, WatchRunRow } from "~/services/dashboardAgentWatchChecks"; +import type { createDashboardAgentWatch as CreateDashboardAgentWatchFunction } from "~/services/dashboardAgentWatches.server"; + +export type DashboardAgentWatchesTestContext = { + prisma: PrismaClient; + agentDb: DashboardAgentDb; + canAccess: boolean; + actor: undefined | { userId: string; client?: string; environmentId?: string }; + /** Every task id the suite would have triggered for real. */ + triggered: string[]; +}; + +async function seedDashboardAgentWatchTestData(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const user = await prisma.user.create({ + data: { email: `${slug}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + await prisma.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: `proj_${slug}` }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 6)}`, + }, + }); + return { user, organization, project, environment }; +} + +export type Seeded = Awaited>; + +export const RUN_START: WatchSpec = { + kind: "run_start", + runId: "run_1", + checkEveryMinutes: 1, + maxHours: 2, + note: "tell me when it starts", +}; + +export const BACKLOG: WatchSpec = { + kind: "backlog_drain", + queue: "task/my-task", + checkEveryMinutes: 5, + maxHours: 2, + note: "tell me when it drains", +}; + +export const HEALTH: WatchSpec = { + kind: "health_recovery", + report: "health", + fromSeverity: "warn", + checkEveryMinutes: 5, + maxHours: 6, + note: "tell me when health recovers", +}; + +/** A run that exists for target validation and is gone when the immediate check reads it. */ +export function readRunOnce(first: WatchRunRow) { + let calls = 0; + return async () => (calls++ === 0 ? first : null); +} + +/** A configured card, with both follow-ups off unless a test turns one on. */ +export function draftFor( + spec: WatchSpec, + followUp: Partial = {} +): WatchDraft { + return { + spec, + followUp: { investigateOnAttention: false, notifyExternally: false, ...followUp }, + }; +} + +type CreateDashboardAgentWatch = typeof CreateDashboardAgentWatchFunction; + +export class DashboardAgentWatchesTestHarness { + private agentDbClient: DashboardAgentDbClient | undefined; + + constructor( + private readonly ctx: DashboardAgentWatchesTestContext, + private readonly createDashboardAgentWatch: CreateDashboardAgentWatch + ) {} + + reset() { + this.ctx.canAccess = true; + this.ctx.actor = undefined; + this.ctx.triggered.length = 0; + } + + async boot(prisma: PrismaClient, connectionUri: string) { + this.ctx.prisma = prisma; + await applyDashboardAgentMigrations((statement) => prisma.$executeRawUnsafe(statement)); + // A pool, not a single connection: concurrent-create tests need the advisory lock to span connections. + this.agentDbClient = createDashboardAgentDb(connectionUri, { max: 8 }); + this.ctx.agentDb = this.agentDbClient.db; + } + + async close() { + await this.agentDbClient?.close(); + this.agentDbClient = undefined; + } + + seed(prisma: PrismaClient, slugBase: string) { + return seedDashboardAgentWatchTestData(prisma, slugBase); + } + + authenticated(seeded: Seeded) { + return { + id: seeded.environment.id, + organizationId: seeded.organization.id, + projectId: seeded.project.id, + slug: "prod", + type: "PRODUCTION", + project: { id: seeded.project.id, externalRef: seeded.project.externalRef }, + organization: { id: seeded.organization.id, slug: seeded.organization.slug }, + } as any; + } + + async seedChat(seeded: Seeded, chatId = "chat_1") { + await createChat(this.ctx.agentDb, { + id: chatId, + organizationId: seeded.organization.id, + userId: seeded.user.id, + }); + return chatId; + } + + runRow(overrides: Partial = {}): WatchRunRow { + return { + friendlyId: "run_1", + status: "PENDING", + queue: "task/my-task", + createdAt: new Date(), + queuedAt: null, + startedAt: null, + completedAt: null, + delayUntil: null, + ...overrides, + }; + } + + /** Injected readers. Defaults keep every condition pending with a live target. */ + fakeCheckDeps(overrides: Partial = {}): WatchCheckDeps { + return { + readRun: async () => this.runRow(), + queueExists: async () => true, + readQueueDepth: async () => ({ depth: 7, source: "live_queue", current: true }), + readQueueOldestAge: async () => ({ + ageMs: 30_000, + source: "live_queue", + current: true, + }), + readErrorRecurrence: async () => null, + readHealth: async () => ({ trustworthy: true, severity: "warn" }), + ...overrides, + }; + } + + create(args: { + seeded: Seeded; + spec?: WatchSpec; + chatId?: string; + environmentId?: string; + investigateOnAttention?: boolean; + watchId?: string; + checkDeps?: Partial; + scheduled?: Array<{ watchId: string; token: string; tick: number }>; + onSchedule?: () => void; + }) { + const environment = this.authenticated(args.seeded); + return this.createDashboardAgentWatch({ + environment: args.environmentId ? { ...environment, id: args.environmentId } : environment, + userId: args.seeded.user.id, + chatId: args.chatId ?? "chat_1", + spec: args.spec ?? RUN_START, + investigateOnAttention: args.investigateOnAttention, + watchId: args.watchId, + deps: { + configured: () => true, + checkDeps: () => this.fakeCheckDeps(args.checkDeps), + scheduleTick: async (params) => { + args.onSchedule?.(); + args.scheduled?.push({ + watchId: params.watchId, + token: params.token, + tick: params.tick, + }); + }, + }, + }); + } + + storedMessages(seeded: Seeded, chatId: string) { + return getChatMessages(this.ctx.agentDb, { + chatId, + userId: seeded.user.id, + organizationId: seeded.organization.id, + }) as Promise | null>; + } +} diff --git a/apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts b/apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts new file mode 100644 index 000000000..0edff7b65 --- /dev/null +++ b/apps/webapp/test/helpers/envConcurrencyLimitPauseTestHelpers.ts @@ -0,0 +1,94 @@ +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RedisOptions } from "ioredis"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { + createRuntimeEnvironment, + createTestOrgProjectWithMember, + uniqueId, +} from "../fixtures/environmentVariablesFixtures"; + +export type EnvConcurrencyLimitPauseTestEngine = RunEngine; + +export function createEnvConcurrencyLimitPauseTestEngine( + prisma: PrismaClient, + redisOptions: RedisOptions +) { + return new RunEngine({ + prisma, + worker: { redis: redisOptions, disabled: true }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + ttlSystem: { disabled: true }, + }, + batchQueue: { redis: redisOptions, consumerEnabled: false }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +// The import chain reaches module-level singletons that throw at load time when +// REDIS_HOST/REDIS_PORT are unset (autoIncrementCounter via triggerTaskV1), so the env must point +// at the redis container BEFORE the modules are imported. Vitest runs each file in its own fork, +// so the env mutation cannot leak into other suites. +export async function loadEnvConcurrencyLimitPauseServices(redisOptions: RedisOptions) { + process.env.REDIS_HOST = redisOptions.host; + process.env.REDIS_PORT = String(redisOptions.port); + process.env.REDIS_TLS_DISABLED = "true"; + const [{ updateEnvConcurrencyLimits }, { PauseEnvironmentService }, runtimeEnvironment] = + await Promise.all([ + import("~/v3/runQueue.server"), + import("~/v3/services/pauseEnvironment.server"), + import("~/models/runtimeEnvironment.server"), + ]); + + return { + updateEnvConcurrencyLimits, + PauseEnvironmentService, + authIncludeBase: runtimeEnvironment.authIncludeBase, + toAuthenticated: runtimeEnvironment.toAuthenticated, + }; +} + +export type EnvConcurrencyLimitPauseServices = Awaited< + ReturnType +>; + +export async function authEnv( + loaded: EnvConcurrencyLimitPauseServices, + prisma: PrismaClient, + environmentId: string +): Promise { + const row = await prisma.runtimeEnvironment.findFirstOrThrow({ + where: { id: environmentId }, + include: loaded.authIncludeBase, + }); + + return loaded.toAuthenticated(row); +} + +export async function seedProductionEnv(prisma: PrismaClient, maximumConcurrencyLimit: number) { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + await prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { maximumConcurrencyLimit }, + }); + + return { organization, project, environment }; +} diff --git a/apps/webapp/vitest.e2e.config.ts b/apps/webapp/vitest.e2e.config.ts index 905c4ac6f..9cbf79c59 100644 --- a/apps/webapp/vitest.e2e.config.ts +++ b/apps/webapp/vitest.e2e.config.ts @@ -1,8 +1,10 @@ import { defineConfig } from "vitest/config"; +import { DurationShardingSequencer } from "@internal/testcontainers/sequencer"; import tsconfigPaths from "vite-tsconfig-paths"; export default defineConfig({ test: { + sequence: { sequencer: DurationShardingSequencer }, include: ["test/**/*.e2e.test.ts"], globals: true, pool: "forks", diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index e07d668f3..ccfb60ca4 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -116,7 +116,9 @@ export class RunEngine { private heartbeatTimeouts: HeartbeatTimeouts; private repairSnapshotTimeoutMs: number; private batchQueue: BatchQueue; + private batchQueueConsumersEnabled: boolean; private workerQueueObserverAbortController?: AbortController; + private quitPromise?: Promise; prisma: PrismaClient; readOnlyPrisma: PrismaReplicaClient; @@ -461,14 +463,14 @@ export class RunEngine { waitpointSystem: this.waitpointSystem, }); - // Initialize BatchQueue for DRR-based batch processing (if configured) - const startBatchQueueConsumers = options.batchQueue?.consumerEnabled ?? true; + // Initialize BatchQueue for DRR-based batch processing. Consumers start lazily when the + // process-item callback is registered; before that they cannot perform useful work. + this.batchQueueConsumersEnabled = options.batchQueue?.consumerEnabled ?? true; + const batchQueueRedis = options.batchQueue?.redis ?? options.queue.redis; this.batchQueue = new BatchQueue({ - redis: { - keyPrefix: `${options.batchQueue?.redis.keyPrefix ?? ""}batch-queue:`, - ...options.batchQueue?.redis, - }, + // Preserve the configured namespace so existing batch state remains addressable. + redis: batchQueueRedis, drr: { quantum: options.batchQueue?.drr?.quantum ?? 5, maxDeficit: options.batchQueue?.drr?.maxDeficit ?? 50, @@ -481,7 +483,7 @@ export class RunEngine { defaultConcurrency: options.batchQueue?.defaultConcurrency ?? 10, globalRateLimiter: options.batchQueue?.globalRateLimiter, workerQueueMaxDepth: options.batchQueue?.workerQueueMaxDepth, - startConsumers: startBatchQueueConsumers, + startConsumers: false, retry: options.batchQueue?.retry, tracer: options.tracer, meter: options.meter, @@ -491,7 +493,7 @@ export class RunEngine { consumerCount: options.batchQueue?.consumerCount ?? 2, drrQuantum: options.batchQueue?.drr?.quantum ?? 5, defaultConcurrency: options.batchQueue?.defaultConcurrency ?? 10, - consumersEnabled: startBatchQueueConsumers, + consumersEnabled: this.batchQueueConsumersEnabled, }); this.runAttemptSystem = new RunAttemptSystem({ @@ -1922,6 +1924,9 @@ export class RunEngine { */ setBatchProcessItemCallback(callback: ProcessBatchItemCallback): void { this.batchQueue.onProcessItem(callback); + if (this.batchQueueConsumersEnabled) { + this.batchQueue.start(); + } } /** @@ -2366,26 +2371,57 @@ export class RunEngine { } } - async quit() { - try { - this.workerQueueObserverAbortController?.abort(); + quit(): Promise { + this.quitPromise ??= this.#quit(); + return this.quitPromise; + } - await this.runQueue.quit(); - await this.worker.stop(); - await this.ttlWorker.stop(); - await this.runLock.quit(); + async #quit(): Promise { + this.workerQueueObserverAbortController?.abort(); - // This is just a failsafe - await this.runLockRedis.quit(); + // Stop resources that actively process work before closing support resources they may use. + const processingResults = await Promise.allSettled([ + this.runQueue.quit(), + this.worker.stop(), + this.ttlWorker.stop(), + this.batchQueue.close(), + ]); + this.#logShutdownFailures( + ["runQueue.quit", "worker.stop", "ttlWorker.stop", "batchQueue.close"], + processingResults + ); - await this.batchQueue.close(); + const supportResults = await Promise.allSettled([ + this.runLock.quit(), + this.debounceSystem.quit(), + ]); + this.#logShutdownFailures(["runLock.quit", "debounceSystem.quit"], supportResults); - await this.debounceSystem.quit(); - } catch (_error) { - // Best-effort shutdown; ignore quit/close errors. + // RunLocker/Redlock owns this client and normally closes it. Do not send a second QUIT, + // but force-disconnect if Redlock failed to leave the connection in its terminal state. + if (this.runLockRedis.status !== "end") { + try { + this.runLockRedis.disconnect(); + } catch (error) { + this.logger.error("RunEngine shutdown operation failed", { + operation: "runLockRedis.disconnect", + error, + }); + } } } + #logShutdownFailures(operations: string[], results: PromiseSettledResult[]): void { + results.forEach((result, index) => { + if (result.status === "rejected") { + this.logger.error("RunEngine shutdown operation failed", { + operation: operations[index], + error: result.reason, + }); + } + }); + } + async repairEnvironment(environment: AuthenticatedEnvironment, dryRun: boolean) { const runIds = await this.runQueue.getCurrentConcurrencyOfEnvironment(environment); diff --git a/internal-packages/run-engine/src/engine/tests/shutdown.test.ts b/internal-packages/run-engine/src/engine/tests/shutdown.test.ts new file mode 100644 index 000000000..a0d421e3c --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/shutdown.test.ts @@ -0,0 +1,94 @@ +import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis"; +import { containerTestWithIsolatedRedisNoClickhouse } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { Logger } from "@trigger.dev/core/logger"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; + +async function connectedClientCount(redis: Redis): Promise { + const clientsInfo = await redis.info("clients"); + const match = clientsInfo.match(/^connected_clients:(\d+)$/m); + + if (!match) { + throw new Error("Redis INFO clients response did not include connected_clients"); + } + + return Number(match[1]); +} + +function engineOptions(redisOptions: RedisOptions) { + // Keep caches and consumers lazy so every connection opened by this test belongs to a shutdown + // resource. The run-lock client remains eager to exercise Redlock's ownership of it. + const lazyRedisOptions = { ...redisOptions, lazyConnect: true }; + + return { + worker: { + disabled: true, + redis: lazyRedisOptions, + workers: 1, + tasksPerWorker: 1, + pollIntervalMs: 10, + immediatePollIntervalMs: 10, + shutdownTimeoutMs: 30_000, + }, + queue: { + redis: lazyRedisOptions, + masterQueueConsumersDisabled: true, + ttlSystem: { disabled: true }, + logLevel: "error" as const, + }, + runLock: { redis: redisOptions }, + cache: { redis: lazyRedisOptions }, + debounce: { redis: lazyRedisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("run-engine-shutdown-test", "0.0.0"), + logger: new Logger("run-engine-shutdown-test", "error"), + }; +} + +describe("RunEngine.quit", () => { + containerTestWithIsolatedRedisNoClickhouse( + "is concurrency-safe, repeatable, and returns Redis connections to baseline", + { timeout: 60_000 }, + async ({ prisma, redisOptions }) => { + await prisma.$queryRaw`SELECT 1`; + + const observer = createRedisClient(redisOptions); + await observer.ping(); + const baselineConnections = await connectedClientCount(observer); + const engine = new RunEngine({ prisma, ...engineOptions(redisOptions) }); + + try { + await expect + .poll(() => connectedClientCount(observer)) + .toBeGreaterThan(baselineConnections); + + const firstQuit = engine.quit(); + const concurrentQuit = engine.quit(); + expect(concurrentQuit).toBe(firstQuit); + + await Promise.all([firstQuit, concurrentQuit, engine.quit()]); + + const repeatedQuit = engine.quit(); + expect(repeatedQuit).toBe(firstQuit); + await repeatedQuit; + + await expect.poll(() => connectedClientCount(observer)).toBe(baselineConnections); + } finally { + await engine.quit(); + await observer.quit(); + } + } + ); +}); diff --git a/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts b/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts index edf5b447c..18517d124 100644 --- a/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts +++ b/packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts @@ -211,6 +211,33 @@ class TestFairQueueHelper { describe("FairQueue", () => { let keys: FairQueueKeyProducer; + describe("worker queue lifecycle", () => { + redisTest( + "aborts a blocking pop without reconnecting the disconnected client", + { timeout: 5000 }, + async ({ redisOptions }) => { + const workerQueue = new WorkerQueueManager({ + redis: redisOptions, + keys: new DefaultFairQueueKeyProducer({ prefix: "abort-test" }), + }); + const abortController = new AbortController(); + + try { + const pop = workerQueue.blockingPop(TEST_WORKER_QUEUE_ID, 60, abortController.signal); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const startedAt = performance.now(); + abortController.abort(); + + await expect(pop).resolves.toBeNull(); + expect(performance.now() - startedAt).toBeLessThan(1000); + } finally { + await workerQueue.close(); + } + } + ); + }); + describe("basic enqueue and process", () => { redisTest( "should enqueue and process a single message", diff --git a/packages/redis-worker/src/fair-queue/workerQueue.ts b/packages/redis-worker/src/fair-queue/workerQueue.ts index b3b75e9db..087c6aedf 100644 --- a/packages/redis-worker/src/fair-queue/workerQueue.ts +++ b/packages/redis-worker/src/fair-queue/workerQueue.ts @@ -153,9 +153,11 @@ export class WorkerQueueManager { if (cleanup && signal) { signal.removeEventListener("abort", cleanup); } - await blockingClient.quit().catch(() => { - // Ignore quit errors (may already be disconnected) - }); + if (blockingClient.status !== "end") { + await blockingClient.quit().catch(() => { + // Ignore quit errors (may already be disconnected) + }); + } } } diff --git a/packages/redis-worker/src/worker.test.ts b/packages/redis-worker/src/worker.test.ts index f5659f379..664a45873 100644 --- a/packages/redis-worker/src/worker.test.ts +++ b/packages/redis-worker/src/worker.test.ts @@ -4,7 +4,22 @@ import { describe } from "node:test"; import { expect } from "vitest"; import { z } from "zod"; import { Worker } from "./worker.js"; -import { createRedisClient } from "@internal/redis"; +import { createRedisClient, type Redis } from "@internal/redis"; + +async function connectedClientCount(redis: Redis): Promise { + const clientsInfo = await redis.info("clients"); + const match = clientsInfo.match(/^connected_clients:(\d+)$/m); + + if (!match) { + throw new Error("Redis INFO clients response did not include connected_clients"); + } + + return Number(match[1]); +} + +function activeTimeoutCount(): number { + return process.getActiveResourcesInfo().filter((resource) => resource === "Timeout").length; +} describe("Worker", () => { redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => { @@ -549,6 +564,58 @@ describe("Worker", () => { } ); + redisTest( + "clears its shutdown deadline and closes Redis connections after a prompt stop", + { timeout: 30_000 }, + async ({ redisContainer }) => { + const redisOptions = { + host: redisContainer.getHost(), + port: redisContainer.getPort(), + password: redisContainer.getPassword(), + }; + const observer = createRedisClient(redisOptions); + await observer.ping(); + + const baselineConnections = await connectedClientCount(observer); + const worker = new Worker({ + name: "shutdown-lifecycle-worker", + redisOptions, + catalog: { + testJob: { + schema: z.object({ value: z.number() }), + visibilityTimeoutMs: 5000, + }, + }, + jobs: { + testJob: async () => {}, + }, + concurrency: { workers: 1, tasksPerWorker: 1 }, + pollIntervalMs: 10, + immediatePollIntervalMs: 10, + shutdownTimeoutMs: 30_000, + logger: new Logger("shutdown-lifecycle-test", "error"), + }).start(); + + try { + await expect + .poll(() => connectedClientCount(observer)) + .toBeGreaterThan(baselineConnections); + + // Let the worker enter its polling loop so the loop, rather than the deadline, wins shutdown. + await new Promise((resolve) => setTimeout(resolve, 50)); + const baselineTimeouts = activeTimeoutCount(); + await worker.stop(); + + await expect.poll(() => connectedClientCount(observer)).toBe(baselineConnections); + await new Promise((resolve) => setImmediate(resolve)); + expect(activeTimeoutCount()).toBeLessThanOrEqual(baselineTimeouts); + } finally { + await worker.stop(); + await observer.quit(); + } + } + ); + redisTest( "Should allow cancelling a job before it's enqueued, but only if the enqueue.cancellationKey is provided", { timeout: 30_000 }, diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts index 64268a1c9..0db651a54 100644 --- a/packages/redis-worker/src/worker.ts +++ b/packages/redis-worker/src/worker.ts @@ -1198,16 +1198,26 @@ class Worker { this.isShuttingDown = true; this.logger.log("Shutting down worker loops...", { signal }); - // Wait for all worker loops to finish. - await Promise.race([ - Promise.all(this.workerLoops), - Worker.delay(this.shutdownTimeoutMs).then(() => { + // Wait for all worker loops to finish, retaining ownership of the deadline timer so the + // losing timeout cannot keep the process alive after a prompt shutdown. + let shutdownDeadline: ReturnType | undefined; + const deadlinePromise = new Promise((resolve) => { + shutdownDeadline = setTimeout(() => { this.logger.error("Worker shutdown timed out", { signal, shutdownTimeoutMs: this.shutdownTimeoutMs, }); - }), - ]); + resolve(); + }, this.shutdownTimeoutMs); + }); + + try { + await Promise.race([Promise.all(this.workerLoops), deadlinePromise]); + } finally { + if (shutdownDeadline) { + clearTimeout(shutdownDeadline); + } + } await this.subscriber?.unsubscribe(); await this.subscriber?.quit(); diff --git a/test-timings.json b/test-timings.json index 33fa9c5f3..7f45aa170 100644 --- a/test-timings.json +++ b/test-timings.json @@ -1,269 +1,544 @@ { - "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 26, - "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2091, - "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 49515, - "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 132479, - "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 132404, - "apps/webapp/app/utils/friendlyId.test.ts": 59, - "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 25, - "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 3372, - "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 36, - "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 477, - "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 27, - "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 6305, - "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 24, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 546, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 511, - "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 6073, - "apps/webapp/app/v3/runStore.server.test.ts": 8480, - "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 5922, - "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 83, - "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 4096, - "apps/webapp/test/GCRARateLimiter.test.ts": 4787, - "apps/webapp/test/SpanPresenter.readthrough.test.ts": 9668, - "apps/webapp/test/activitySeries.server.test.ts": 25, - "apps/webapp/test/aiTitleRateLimiter.test.ts": 744, - "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 97711, - "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 179054, - "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5727, - "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 5378, - "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 11836, - "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3476, - "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 8669, - "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 12490, - "apps/webapp/test/apiRunListPresenter.test.ts": 148237, - "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 6843, - "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3321, - "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 9115, - "apps/webapp/test/batchListPresenter.readroute.test.ts": 11138, - "apps/webapp/test/batchPresenter.test.ts": 16534, - "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 1959, - "apps/webapp/test/batchRunAccess.test.ts": 8353, - "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6556, - "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 1037, - "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 5624, - "apps/webapp/test/billingAlertsFormat.test.ts": 34, - "apps/webapp/test/billingLimit.schemas.test.ts": 40, - "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 16155, - "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3424, - "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 21, - "apps/webapp/test/billingLimitConvergeResolve.test.ts": 200, - "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 609, - "apps/webapp/test/billingLimitHit.test.ts": 25, - "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 35, - "apps/webapp/test/billingLimitQueuedRuns.test.ts": 12281, - "apps/webapp/test/billingLimitReconcileTick.test.ts": 213, - "apps/webapp/test/billingLimitReconciliation.test.ts": 630, - "apps/webapp/test/billingLimitResolve.test.ts": 17, - "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 26, - "apps/webapp/test/billingLimitsRoute.test.ts": 1102, - "apps/webapp/test/branchableEnvironment.test.ts": 20, - "apps/webapp/test/bufferedTriggerPayload.test.ts": 25, - "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6183, - "apps/webapp/test/calculateNextSchedule.test.ts": 208, - "apps/webapp/test/chartActivityTimeAxis.test.ts": 29, - "apps/webapp/test/chartXAxisTicks.test.ts": 29, - "apps/webapp/test/chartZoomRange.test.ts": 22, - "apps/webapp/test/chat-snapshot-integration.test.ts": 1896, - "apps/webapp/test/checkPermissions.test.ts": 22, - "apps/webapp/test/clickhouseFactory.test.ts": 3978, - "apps/webapp/test/components/DateTime.test.ts": 784, - "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 136, - "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 273, - "apps/webapp/test/components/runs/v3/RunTag.test.ts": 190, - "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 339, - "apps/webapp/test/computeBucket.test.ts": 123, - "apps/webapp/test/computeMigration.test.ts": 22, - "apps/webapp/test/concurrentFlushScheduler.test.ts": 716, - "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 8603, - "apps/webapp/test/crossSeamGuard.proof.test.ts": 5538, - "apps/webapp/test/dependentAttemptScope.test.ts": 17, - "apps/webapp/test/detectQueryTables.test.ts": 259, - "apps/webapp/test/detectbadJsonStrings.test.ts": 73, - "apps/webapp/test/devBranchServices.test.ts": 4029, - "apps/webapp/test/devPresenceRecency.test.ts": 1179, - "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2678, - "apps/webapp/test/duplicateTaskIds.test.ts": 20, - "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1730, - "apps/webapp/test/emailPattern.test.ts": 18, - "apps/webapp/test/engine/batchPayloads.test.ts": 5365, - "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 10574, - "apps/webapp/test/engine/streamBatchItems.test.ts": 19344, - "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 4014, - "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 133741, - "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 91796, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 134153, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133523, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 173067, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 173296, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 172996, - "apps/webapp/test/engine/triggerTask.test.ts": 92296, - "apps/webapp/test/environmentSort.test.ts": 29, - "apps/webapp/test/environmentVariableDeduplication.test.ts": 24, - "apps/webapp/test/environmentVariableRules.test.ts": 19, - "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3690, - "apps/webapp/test/environmentVariablesRepository.test.ts": 4145, - "apps/webapp/test/errorFingerprinting.test.ts": 28, - "apps/webapp/test/errorGroupWebhook.test.ts": 59, - "apps/webapp/test/findEnvironmentByApiKey.test.ts": 3764, - "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5555, - "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 8848, - "apps/webapp/test/getDeploymentImageRef.test.ts": 306, - "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6408, - "apps/webapp/test/googleEmailVerification.test.ts": 27, - "apps/webapp/test/httpErrors.test.ts": 39, - "apps/webapp/test/idempotencyDedupResidency.test.ts": 9623, - "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 5778, - "apps/webapp/test/inviteRoleLadder.test.ts": 18, - "apps/webapp/test/member.server.test.ts": 5194, - "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 274, - "apps/webapp/test/mfaRateLimiter.test.ts": 632, - "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 850, - "apps/webapp/test/mollifierClaimResolution.test.ts": 469, - "apps/webapp/test/mollifierDecisionLabels.test.ts": 47, - "apps/webapp/test/mollifierDrainerHandler.test.ts": 450, - "apps/webapp/test/mollifierDrainerWorker.test.ts": 2020, - "apps/webapp/test/mollifierDrainingGauge.test.ts": 716, - "apps/webapp/test/mollifierGate.test.ts": 492, - "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 423, - "apps/webapp/test/mollifierMollify.test.ts": 223, - "apps/webapp/test/mollifierMutateWithFallback.test.ts": 445, - "apps/webapp/test/mollifierReadFallback.test.ts": 405, - "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 249, - "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 522, - "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 448, - "apps/webapp/test/mollifierStaleSweep.test.ts": 1047, - "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 559, - "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 25, - "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 815, - "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 20, - "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 21, - "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 197, - "apps/webapp/test/mollifierSyntheticTrace.test.ts": 303, - "apps/webapp/test/mollifierTripEvaluator.test.ts": 831, - "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 30311, - "apps/webapp/test/objectStore.test.ts": 5520, - "apps/webapp/test/orgBanner.test.ts": 18, - "apps/webapp/test/organizationDataStoresRegistry.test.ts": 5455, - "apps/webapp/test/otlpExporter.test.ts": 144, - "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 5831, - "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 317, - "apps/webapp/test/pauseEnvironment.server.test.ts": 8847, - "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 6886, - "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 16145, - "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 6231, - "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 36865, - "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 21, - "apps/webapp/test/prismaErrors.test.ts": 219, - "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3592, - "apps/webapp/test/promptOverrideSource.test.ts": 17, - "apps/webapp/test/queryResultsTimeTicks.test.ts": 650, - "apps/webapp/test/queueListPagination.test.ts": 31, - "apps/webapp/test/rbacFallbackBranch.test.ts": 3600, - "apps/webapp/test/realtime/boundedTtlCache.test.ts": 22, - "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 41562, - "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 68, - "apps/webapp/test/realtime/envChangeRouter.test.ts": 1176, - "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4496, - "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 342, - "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 607, - "apps/webapp/test/realtime/replayCursorStore.test.ts": 1529, - "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 584, - "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3540, - "apps/webapp/test/realtime/runReaderProjection.test.ts": 53, - "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 7242, - "apps/webapp/test/realtime/shadowCompare.test.ts": 26, - "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5726, - "apps/webapp/test/redisRealtimeStreams.test.ts": 5745, - "apps/webapp/test/registryConfig.test.ts": 343, - "apps/webapp/test/reloadingRegistry.test.ts": 362, - "apps/webapp/test/removeTeamMember.test.ts": 9473, - "apps/webapp/test/replay-after-crash.test.ts": 2002, - "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 4349, - "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5698, - "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6634, - "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 5232, - "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 6025, - "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 550, - "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 5799, - "apps/webapp/test/runEngineHandlers.test.ts": 14088, - "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 348, - "apps/webapp/test/runOpsDbTopology.test.ts": 4214, - "apps/webapp/test/runOpsMintCutover.test.ts": 3808, - "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3407, - "apps/webapp/test/runOpsSplitMode.test.ts": 4202, - "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 473, - "apps/webapp/test/runOpsSplitReadGate.test.ts": 18, - "apps/webapp/test/runPresenterReadRoute.test.ts": 3909, - "apps/webapp/test/runsBackfiller.test.ts": 8095, - "apps/webapp/test/runsReplicationInstance.test.ts": 24640, - "apps/webapp/test/runsReplicationService.part1.test.ts": 25612, - "apps/webapp/test/runsReplicationService.part2.test.ts": 22323, - "apps/webapp/test/runsReplicationService.part3.test.ts": 12303, - "apps/webapp/test/runsReplicationService.part4.test.ts": 27040, - "apps/webapp/test/runsReplicationService.part5.test.ts": 9175, - "apps/webapp/test/runsReplicationService.part6.test.ts": 12682, - "apps/webapp/test/runsReplicationService.part7.test.ts": 69508, - "apps/webapp/test/runsReplicationService.part8.test.ts": 24479, - "apps/webapp/test/runsReplicationService.part9.test.ts": 12463, - "apps/webapp/test/runsRepository.part1.test.ts": 22987, - "apps/webapp/test/runsRepository.part2.test.ts": 23786, - "apps/webapp/test/runsRepository.part3.test.ts": 18531, - "apps/webapp/test/runsRepository.part4.test.ts": 24073, - "apps/webapp/test/runsRepository.readthrough.test.ts": 39432, - "apps/webapp/test/runsRepositoryCpres.test.ts": 8672, - "apps/webapp/test/runsRepositoryCursor.test.ts": 28582, - "apps/webapp/test/safeEnvironmentLog.test.ts": 17, - "apps/webapp/test/safeIntegrationLog.test.ts": 15, - "apps/webapp/test/safeRequestLogContext.test.ts": 26, - "apps/webapp/test/safeWebhookFetch.test.ts": 199, - "apps/webapp/test/safeWebhookUrl.test.ts": 30, - "apps/webapp/test/sameOriginNavigation.test.ts": 53, - "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 30, - "apps/webapp/test/sanitizeUrl.test.ts": 22, - "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 280, - "apps/webapp/test/sentryTenantContext.test.ts": 23, - "apps/webapp/test/sentryTraceContext.server.test.ts": 70, - "apps/webapp/test/services.controlPlane.readthrough.test.ts": 5401, - "apps/webapp/test/services/organizationAccessToken.test.ts": 241, - "apps/webapp/test/services/personalAccessToken.test.ts": 232, - "apps/webapp/test/sessionDuration.test.ts": 10233, - "apps/webapp/test/sessions.readthrough.test.ts": 5859, - "apps/webapp/test/sessionsReplicationService.test.ts": 17051, - "apps/webapp/test/shouldRevalidateRunsList.test.ts": 19, - "apps/webapp/test/slackOAuthResultLog.test.ts": 23, - "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5373, - "apps/webapp/test/streamLoader.controlPlane.test.ts": 5162, - "apps/webapp/test/tenantContext.test.ts": 43, - "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 38, - "apps/webapp/test/tenantContextResolver.test.ts": 37, - "apps/webapp/test/timeGranularity.test.ts": 27, - "apps/webapp/test/timelineSpanEvents.test.ts": 25, - "apps/webapp/test/traceExport.test.ts": 29, - "apps/webapp/test/updateMetadata.test.ts": 19006, - "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 7309, - "apps/webapp/test/utils/timezones.test.ts": 31, - "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 5949, - "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 6678, - "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 5388, - "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4587, - "apps/webapp/test/validateGitBranchName.test.ts": 23, - "apps/webapp/test/vercelUrls.test.ts": 18, - "apps/webapp/test/verifyDeploymentImage.test.ts": 922, - "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 7670, - "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 8556, - "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 5386, - "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 8112, - "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 4994, - "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 5687, - "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 35556, - "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5062, - "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 5653, - "apps/webapp/test/webhookErrorAlerts.test.ts": 61, - "apps/webapp/test/workerGroupAccess.test.ts": 33, - "apps/webapp/test/workerQueueSplit.server.test.ts": 23, - "apps/webapp/test/workerQueueSplit.test.ts": 27, - "apps/webapp/test/workerRegions.test.ts": 487, + "apps/webapp/app/components/code/StreamdownRenderer.test.ts": 172, + "apps/webapp/app/components/code/tsql/tsqlLinter.test.ts": 177, + "apps/webapp/app/components/dashboard-agent/DashboardAgentMessages.render.test.ts": 24, + "apps/webapp/app/components/dashboard-agent/InvestigationCard.render.test.ts": 8, + "apps/webapp/app/components/dashboard-agent/InvestigationCard.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/ReportView.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/WatchChips.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/agent-shortcuts.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/ai-entry-points.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/ask-ai-channels.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/askAiOpenRequest.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/chat-layout.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/coalesced-reload.test.ts": 7, + "apps/webapp/app/components/dashboard-agent/composer-escape.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/dashboardAgentOpenRequest.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/demo/demo.test.ts": 18, + "apps/webapp/app/components/dashboard-agent/diagnosis-actions.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/explicit-prompt.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/header-labels.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/investigate-prompts.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/investigation-winners.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/last-chat-storage.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/message-limits.test.ts": 30, + "apps/webapp/app/components/dashboard-agent/message-order.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/message-quota.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/model-markdown.test.ts": 91, + "apps/webapp/app/components/dashboard-agent/navigate-target.test.ts": 9, + "apps/webapp/app/components/dashboard-agent/opened-chat.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/page-label.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/panel-escape.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/pending-intents.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/pending-turn.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/progress-line.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/report-block-adapter.test.ts": 12, + "apps/webapp/app/components/dashboard-agent/report-spark.test.ts": 210, + "apps/webapp/app/components/dashboard-agent/resolve-uris.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/retry-action.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/run-id.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/send-request.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/settled-transcript.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/page-mappers.test.ts": 25, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/promoted.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/suggested-prompts/resolver.test.ts": 47, + "apps/webapp/app/components/dashboard-agent/thinking-marker.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/tool-labels.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/tooltip-accessible-name.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/turn-error.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/turn-navigation.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/turn-teardown.test.ts": 3, + "apps/webapp/app/components/dashboard-agent/unread-counts.test.ts": 5, + "apps/webapp/app/components/dashboard-agent/unread-work.test.ts": 2, + "apps/webapp/app/components/dashboard-agent/view-actions.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/view-blocks.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/view-catalog.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/wake-banner.test.ts": 6, + "apps/webapp/app/components/dashboard-agent/wake-poll.test.ts": 8, + "apps/webapp/app/components/dashboard-agent/watch-activity.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/watch-card-state.test.ts": 4, + "apps/webapp/app/components/dashboard-agent/watch-card.test.ts": 35, + "apps/webapp/app/components/dashboard-agent/watch-chips.test.ts": 3, + "apps/webapp/app/components/queues/queue-name.test.ts": 2, + "apps/webapp/app/components/queues/queue-thresholds.test.ts": 3, + "apps/webapp/app/presenters/v3/reports/report-layout.test.ts": 19, + "apps/webapp/app/routes/storybook.agent-ui/fixtures.test.ts": 7, + "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 3, + "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 3, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 49314, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 130135, + "apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts": 49166, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 131000, + "apps/webapp/app/utils/apiKeys.test.ts": 5, + "apps/webapp/app/utils/boundedRequestBody.server.test.ts": 21, + "apps/webapp/app/utils/cspImageOrigins.test.ts": 5, + "apps/webapp/app/utils/databaseMetrics.server.test.ts": 3, + "apps/webapp/app/utils/deeplinkPages.test.ts": 12, + "apps/webapp/app/utils/environmentAccess.test.ts": 3, + "apps/webapp/app/utils/friendlyId.test.ts": 4, + "apps/webapp/app/utils/impersonationPaths.test.ts": 4, + "apps/webapp/app/utils/impersonationState.test.ts": 2, + "apps/webapp/app/utils/localHostGuard.test.ts": 3, + "apps/webapp/app/utils/logSearch.test.ts": 4, + "apps/webapp/app/utils/nullBytes.test.ts": 3, + "apps/webapp/app/utils/pageSwitching.test.ts": 35, + "apps/webapp/app/utils/pageTitle.test.ts": 5, + "apps/webapp/app/utils/plainCustomerCards.test.ts": 5, + "apps/webapp/app/utils/prismaConnectionUrl.test.ts": 2, + "apps/webapp/app/utils/requestIdempotency.test.ts": 3, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 4, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCoresidencySentinel.server.test.ts": 5, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.dispatchFreshness.test.ts": 4502, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 2898, + "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 5, + "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 5, + "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 7, + "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 7072, + "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 2, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 4, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 6, + "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 5978, + "apps/webapp/app/v3/runStore.server.test.ts": 9961, + "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 5609, + "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 4, + "apps/webapp/app/v3/utils/priority.test.ts": 3, + "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 2984, + "apps/webapp/test/GCRARateLimiter.test.ts": 4553, + "apps/webapp/test/SpanPresenter.readthrough.test.ts": 7526, + "apps/webapp/test/activitySeries.server.test.ts": 4, + "apps/webapp/test/additionalApiKeyIssuance.test.ts": 3, + "apps/webapp/test/aiTitleRateLimiter.test.ts": 165, + "apps/webapp/test/api-auth.e2e.test.ts": 20090, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 93816, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 174685, + "apps/webapp/test/apiAuthActorClaim.test.ts": 4, + "apps/webapp/test/apiAuthScope.test.ts": 21, + "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5924, + "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 5100, + "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 7275, + "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3030, + "apps/webapp/test/apiBuilderAuthorization.test.ts": 2, + "apps/webapp/test/apiKeysPresenter.test.ts": 9299, + "apps/webapp/test/apiRateLimitJwtActor.test.ts": 6, + "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 2556, + "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 7084, + "apps/webapp/test/apiRunListPresenter.readthrough.test.ts": 23000, + "apps/webapp/test/apiRunListPresenter.test.ts": 93935, + "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 7573, + "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3367, + "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 10909, + "apps/webapp/test/authFeatureControls.test.ts": 3, + "apps/webapp/test/authorizationCodeConsent.test.ts": 9295, + "apps/webapp/test/authorizationRateLimitMiddleware.test.ts": 1, + "apps/webapp/test/authorizationRateLimitMiddlewareBypass.test.ts": 221, + "apps/webapp/test/batchListPresenter.readroute.test.ts": 9942, + "apps/webapp/test/batchPresenter.test.ts": 12839, + "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 8, + "apps/webapp/test/batchRunAccess.test.ts": 8683, + "apps/webapp/test/batchServices.replicaLag.test.ts": 4174, + "apps/webapp/test/batchStreamGrants.test.ts": 410, + "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6059, + "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 3, + "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 7158, + "apps/webapp/test/billingAlertsDefaults.test.ts": 2, + "apps/webapp/test/billingAlertsFormat.test.ts": 11, + "apps/webapp/test/billingLimit.schemas.test.ts": 6, + "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 13499, + "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3140, + "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 3, + "apps/webapp/test/billingLimitConvergeResolve.test.ts": 8, + "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 4, + "apps/webapp/test/billingLimitHit.test.ts": 3, + "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 2, + "apps/webapp/test/billingLimitQueuedRuns.test.ts": 18429, + "apps/webapp/test/billingLimitReconcileTick.test.ts": 6, + "apps/webapp/test/billingLimitReconciliation.test.ts": 3178, + "apps/webapp/test/billingLimitResolve.test.ts": 2, + "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 2, + "apps/webapp/test/billingLimitsRoute.test.ts": 21, + "apps/webapp/test/branchableEnvironment.test.ts": 3, + "apps/webapp/test/bufferedTriggerPayload.test.ts": 3, + "apps/webapp/test/bulkActionV2.replicaLag.test.ts": 3510, + "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6198, + "apps/webapp/test/calculateNextSchedule.test.ts": 177, + "apps/webapp/test/cancelRouteReplicaLag.guard.test.ts": 3023, + "apps/webapp/test/cancelSupersededDeployments.test.ts": 8, + "apps/webapp/test/chartActivityTimeAxis.test.ts": 12, + "apps/webapp/test/chartXAxisTicks.test.ts": 9, + "apps/webapp/test/chartZoomRange.test.ts": 2, + "apps/webapp/test/chat-snapshot-integration.test.ts": 3333, + "apps/webapp/test/checkPermissions.test.ts": 3, + "apps/webapp/test/checkSchedule.test.ts": 9949, + "apps/webapp/test/claimTtl.test.ts": 2, + "apps/webapp/test/clickhouseEventRepositoryJsonRecovery.test.ts": 6102, + "apps/webapp/test/clickhouseFactory.test.ts": 3934, + "apps/webapp/test/components/DateTime.test.ts": 13, + "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 10, + "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 109, + "apps/webapp/test/components/runs/v3/RunTag.test.ts": 4, + "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 3, + "apps/webapp/test/components/webhookDeliveries/buildDeliveryTimelineItems.test.ts": 4, + "apps/webapp/test/computeBucket.test.ts": 96, + "apps/webapp/test/computeMigration.test.ts": 4, + "apps/webapp/test/concurrencySystemPercentOverride.test.ts": 4088, + "apps/webapp/test/concurrentFlushScheduler.test.ts": 356, + "apps/webapp/test/contextlessPatRoutes.test.ts": 25, + "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 9165, + "apps/webapp/test/createEnvironmentApiKey.test.ts": 11337, + "apps/webapp/test/crossSeamGuard.proof.test.ts": 5184, + "apps/webapp/test/dashboardAgentAlertAdminPreview.test.ts": 3, + "apps/webapp/test/dashboardAgentBodyCap.test.ts": 106, + "apps/webapp/test/dashboardAgentChatRetention.test.ts": 3030, + "apps/webapp/test/dashboardAgentClientMetadata.test.ts": 18, + "apps/webapp/test/dashboardAgentCreateChatOrdering.test.ts": 12, + "apps/webapp/test/dashboardAgentDurableResume.test.ts": 4269, + "apps/webapp/test/dashboardAgentEvalPolicyAuth.test.ts": 47, + "apps/webapp/test/dashboardAgentForeignChat.test.ts": 6, + "apps/webapp/test/dashboardAgentHeadStart.test.ts": 5, + "apps/webapp/test/dashboardAgentImageCsp.test.ts": 3, + "apps/webapp/test/dashboardAgentInProxyMintFailure.test.ts": 9, + "apps/webapp/test/dashboardAgentInvestigationSettlementCard.test.ts": 3, + "apps/webapp/test/dashboardAgentInvestigationWinner.test.ts": 2, + "apps/webapp/test/dashboardAgentLastReadBackfill.test.ts": 3079, + "apps/webapp/test/dashboardAgentLegacyMessagesColumn.test.ts": 123, + "apps/webapp/test/dashboardAgentMessageCards.test.ts": 9, + "apps/webapp/test/dashboardAgentMessageSurrogate.test.ts": 3020, + "apps/webapp/test/dashboardAgentQueriesTenantIsolation.test.ts": 3954, + "apps/webapp/test/dashboardAgentQuota.test.ts": 3891, + "apps/webapp/test/dashboardAgentRoutes.test.ts": 10, + "apps/webapp/test/dashboardAgentSurrogatePersist.test.ts": 3951, + "apps/webapp/test/dashboardAgentTenantIsolation.test.ts": 4250, + "apps/webapp/test/dashboardAgentToolScopes.test.ts": 4, + "apps/webapp/test/dashboardAgentTranscriptStore.test.ts": 8600, + "apps/webapp/test/dashboardAgentUnreadWorkScope.test.ts": 3104, + "apps/webapp/test/dashboardAgentWakeActivity.test.ts": 3041, + "apps/webapp/test/dashboardAgentWatchAlertEnvironments.test.ts": 3993, + "apps/webapp/test/dashboardAgentWatchAlertFanout.test.ts": 9, + "apps/webapp/test/dashboardAgentWatchAlertGate.test.ts": 3, + "apps/webapp/test/dashboardAgentWatchAlertOwnerScope.test.ts": 3187, + "apps/webapp/test/dashboardAgentWatchBatchFairness.test.ts": 4002, + "apps/webapp/test/dashboardAgentWatchBatchRecording.test.ts": 4161, + "apps/webapp/test/dashboardAgentWatchCardAtomicity.test.ts": 4305, + "apps/webapp/test/dashboardAgentWatchCardRequestId.test.ts": 3168, + "apps/webapp/test/dashboardAgentWatchChecks.test.ts": 16, + "apps/webapp/test/dashboardAgentWatchCreationReads.test.ts": 3, + "apps/webapp/test/dashboardAgentWatchErrorFingerprint.test.ts": 3026, + "apps/webapp/test/dashboardAgentWatchInvestigate.test.ts": 16, + "apps/webapp/test/dashboardAgentWatchLimitStatus.test.ts": 2800, + "apps/webapp/test/dashboardAgentWatchLimits.test.ts": 5568, + "apps/webapp/test/dashboardAgentWatchQueueAge.test.ts": 3, + "apps/webapp/test/dashboardAgentWatchQueueName.test.ts": 4369, + "apps/webapp/test/dashboardAgentWatchSweepAlertOnce.test.ts": 2843, + "apps/webapp/test/dashboardAgentWatchSweepBoundary.test.ts": 3481, + "apps/webapp/test/dashboardAgentWatchTenancy.test.ts": 4237, + "apps/webapp/test/dashboardAgentWatchToken.test.ts": 22, + "apps/webapp/test/dashboardAgentWatchWording.test.ts": 5, + "apps/webapp/test/dashboardAgentWatches.batch.test.ts": 93476, + "apps/webapp/test/dashboardAgentWatches.delivery.test.ts": 17111, + "apps/webapp/test/dashboardAgentWatches.lifecycle.test.ts": 16072, + "apps/webapp/test/deleteTaskSchedule.test.ts": 7573, + "apps/webapp/test/deliveryIdBounds.test.ts": 19, + "apps/webapp/test/dependentAttemptScope.test.ts": 4, + "apps/webapp/test/deploymentApiPaths.test.ts": 2, + "apps/webapp/test/detectQueryTables.test.ts": 159, + "apps/webapp/test/detectbadJsonStrings.test.ts": 58, + "apps/webapp/test/devBranchServices.test.ts": 4992, + "apps/webapp/test/devPresenceRecency.test.ts": 209, + "apps/webapp/test/directorySyncEffects.server.test.ts": 12, + "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2555, + "apps/webapp/test/duplicateTaskIds.test.ts": 2, + "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1623, + "apps/webapp/test/emailPattern.test.ts": 5, + "apps/webapp/test/engine/batchPayloads.test.ts": 5016, + "apps/webapp/test/engine/dequeueWorkerVersionFreshness.test.ts": 49054, + "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 48923, + "apps/webapp/test/engine/streamBatchItems.test.ts": 22321, + "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 4029, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 131643, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 90520, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 213034, + "apps/webapp/test/engine/triggerTask.externalDeploymentId.pending.test.ts": 125439, + "apps/webapp/test/engine/triggerTask.externalDeploymentId.resolution.test.ts": 125439, + "apps/webapp/test/engine/triggerTask.externalDeploymentId.test.ts": 125439, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 171570, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 172152, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 171782, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 171372, + "apps/webapp/test/engine/triggerTask.test.ts": 171950, + "apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts": 3608, + "apps/webapp/test/env.server.test.ts": 573, + "apps/webapp/test/envConcurrencyLimitPause.server.test.ts": 8769, + "apps/webapp/test/envConcurrencyLimitPauseDirect.server.test.ts": 9014, + "apps/webapp/test/envConcurrencyLimitPauseService.server.test.ts": 7489, + "apps/webapp/test/envJwtActorClaim.test.ts": 24, + "apps/webapp/test/envParamRoute.ownership.test.ts": 4, + "apps/webapp/test/environmentSort.test.ts": 9, + "apps/webapp/test/environmentVariableApiAccess.test.ts": 8, + "apps/webapp/test/environmentVariableDeduplication.test.ts": 3, + "apps/webapp/test/environmentVariableRules.test.ts": 3, + "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3912, + "apps/webapp/test/environmentVariablesReplicaRouting.test.ts": 8817, + "apps/webapp/test/environmentVariablesRepository.test.ts": 4096, + "apps/webapp/test/errorFingerprinting.test.ts": 9, + "apps/webapp/test/errorGroupWebhook.test.ts": 6, + "apps/webapp/test/externalDeploymentCache.test.ts": 160, + "apps/webapp/test/featureFlags.test.ts": 4158, + "apps/webapp/test/findEnvironmentByApiKey.test.ts": 5169, + "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5425, + "apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts": 3375, + "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 10112, + "apps/webapp/test/getDeploymentImageRef.test.ts": 7, + "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6140, + "apps/webapp/test/googleEmailVerification.test.ts": 4, + "apps/webapp/test/healthcheck-require-plugins.e2e.test.ts": 37215, + "apps/webapp/test/httpErrors.test.ts": 18, + "apps/webapp/test/idempotencyDedupResidency.test.ts": 5951, + "apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts": 7, + "apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts": 11744, + "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 5913, + "apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts": 5501, + "apps/webapp/test/impersonationConsent.test.ts": 9467, + "apps/webapp/test/internalApiOrigin.test.ts": 2, + "apps/webapp/test/inviteRoleLadder.test.ts": 2, + "apps/webapp/test/logger.server.onError.test.ts": 19, + "apps/webapp/test/logsSearchProjector.test.ts": 9, + "apps/webapp/test/logsSearchProjectorRedisStore.test.ts": 190, + "apps/webapp/test/logsSearchProjectorStateStore.test.ts": 2715, + "apps/webapp/test/member.server.test.ts": 7181, + "apps/webapp/test/memberDevEnvironments.server.test.ts": 5862, + "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 5, + "apps/webapp/test/metadataRouteReplicaLag.guard.test.ts": 7, + "apps/webapp/test/mfaRateLimiter.test.ts": 174, + "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 560, + "apps/webapp/test/mollifierClaimResolution.test.ts": 8, + "apps/webapp/test/mollifierDecisionLabels.test.ts": 5, + "apps/webapp/test/mollifierDrainerHandler.test.ts": 12, + "apps/webapp/test/mollifierDrainerWorker.test.ts": 6, + "apps/webapp/test/mollifierDrainingGauge.test.ts": 448, + "apps/webapp/test/mollifierGate.test.ts": 14, + "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 10, + "apps/webapp/test/mollifierMollify.test.ts": 6, + "apps/webapp/test/mollifierMutateWithFallback.test.ts": 10, + "apps/webapp/test/mollifierReadFallback.test.ts": 14, + "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 2, + "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 8, + "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 5, + "apps/webapp/test/mollifierStaleSweep.test.ts": 482, + "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 4, + "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 8, + "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 196, + "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 4, + "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 3, + "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 6, + "apps/webapp/test/mollifierSyntheticTrace.test.ts": 5, + "apps/webapp/test/mollifierTripEvaluator.test.ts": 155, + "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 24036, + "apps/webapp/test/objectStore.test.ts": 6327, + "apps/webapp/test/orgBanner.test.ts": 3, + "apps/webapp/test/orgMember.server.test.ts": 4296, + "apps/webapp/test/organizationDataStoresRegistry.test.ts": 6996, + "apps/webapp/test/otlpExporter.test.ts": 8, + "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 5680, + "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 81, + "apps/webapp/test/pauseEnvironment.server.test.ts": 9596, + "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 5566, + "apps/webapp/test/platformNotifications.test.ts": 6, + "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 8879, + "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 5563, + "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 31639, + "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 2, + "apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts": 3561, + "apps/webapp/test/prismaErrors.test.ts": 2, + "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3785, + "apps/webapp/test/projectEnvironmentCredentialRoute.test.ts": 10, + "apps/webapp/test/projectEnvironmentsBranchScope.test.ts": 33, + "apps/webapp/test/projectSettingsToastRedirect.test.ts": 13, + "apps/webapp/test/promptOverrideSource.test.ts": 2, + "apps/webapp/test/publicAccessTokenResponse.test.ts": 41, + "apps/webapp/test/publicTokensRoute.test.ts": 3774, + "apps/webapp/test/publishClaimResult.test.ts": 5, + "apps/webapp/test/queryResultsTimeTicks.test.ts": 4, + "apps/webapp/test/queryRouteReadOnly.test.ts": 88, + "apps/webapp/test/queryScope.test.ts": 5, + "apps/webapp/test/queueDepthSeries.test.ts": 3, + "apps/webapp/test/queueListPagination.test.ts": 2, + "apps/webapp/test/queueMetricsMapping.test.ts": 6, + "apps/webapp/test/queueRetrieveJwt.test.ts": 27, + "apps/webapp/test/queueSparklineGrid.test.ts": 9, + "apps/webapp/test/rbacFallbackBranch.test.ts": 7170, + "apps/webapp/test/rbacFallbackSessionFloor.test.ts": 4551, + "apps/webapp/test/reacquireClearedGlobalWinner.test.ts": 5, + "apps/webapp/test/readBodyWithCap.test.ts": 15, + "apps/webapp/test/readRunForEvent.replicaLag.test.ts": 8719, + "apps/webapp/test/realtime/boundedTtlCache.test.ts": 4, + "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 42422, + "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 7, + "apps/webapp/test/realtime/envChangeRouter.test.ts": 942, + "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4057, + "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 6, + "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 287, + "apps/webapp/test/realtime/replayCursorStore.test.ts": 1134, + "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 372, + "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3207, + "apps/webapp/test/realtime/runReaderProjection.test.ts": 3, + "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 8141, + "apps/webapp/test/realtime/shadowCompare.test.ts": 4, + "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5721, + "apps/webapp/test/realtimeClient.test.ts": 1, + "apps/webapp/test/realtimeServices.replicaLag.test.ts": 4498, + "apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts": 3098, + "apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts": 6331, + "apps/webapp/test/realtimeStreamsVersion.test.ts": 3, + "apps/webapp/test/redisRealtimeStreams.test.ts": 5391, + "apps/webapp/test/registryConfig.test.ts": 378, + "apps/webapp/test/reloadingRegistry.test.ts": 3, + "apps/webapp/test/removeTeamMember.test.ts": 10573, + "apps/webapp/test/replay-after-crash.test.ts": 4057, + "apps/webapp/test/replayRouteReplicaLag.guard.test.ts": 5527, + "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 3452, + "apps/webapp/test/reportCurationTrust.test.ts": 19, + "apps/webapp/test/reportHealth.test.ts": 23, + "apps/webapp/test/reportHealthData.test.ts": 12, + "apps/webapp/test/reportMetricDelta.test.ts": 12, + "apps/webapp/test/reportPresenter.test.ts": 26, + "apps/webapp/test/reportRenderParity.test.ts": 53, + "apps/webapp/test/reportTrust.test.ts": 4, + "apps/webapp/test/reportsApiRoute.test.ts": 14, + "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5575, + "apps/webapp/test/resolveBatchForRealtime.test.ts": 3, + "apps/webapp/test/resolveExternalIdReuse.test.ts": 10008, + "apps/webapp/test/resolveOrgIdFromSlugForUser.test.ts": 2918, + "apps/webapp/test/resolveProjectScopedEnvironments.test.ts": 3, + "apps/webapp/test/resolveTriggerUri.test.ts": 8, + "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6353, + "apps/webapp/test/routeCspImgSrc.test.ts": 69, + "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 6057, + "apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts": 7087, + "apps/webapp/test/runCommitAuthorization.test.ts": 8, + "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 5175, + "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 7, + "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 4964, + "apps/webapp/test/runEngineHandlers.test.ts": 16338, + "apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts": 8602, + "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 5, + "apps/webapp/test/runOpsDbTopology.test.ts": 6510, + "apps/webapp/test/runOpsMintCutover.test.ts": 3355, + "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3080, + "apps/webapp/test/runOpsSplitMode.test.ts": 4077, + "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 3, + "apps/webapp/test/runOpsSplitReadGate.test.ts": 4, + "apps/webapp/test/runPresenterReadRoute.test.ts": 4214, + "apps/webapp/test/runPresenters.replicaLag.test.ts": 4893, + "apps/webapp/test/runTimestamps.test.ts": 4, + "apps/webapp/test/runsBackfiller.test.ts": 9029, + "apps/webapp/test/runsReplicationBenchmark.test.ts": 1, + "apps/webapp/test/runsReplicationInstance.test.ts": 19226, + "apps/webapp/test/runsReplicationJsonRecoveryBenchmark.test.ts": 1, + "apps/webapp/test/runsReplicationService.part1.test.ts": 29667, + "apps/webapp/test/runsReplicationService.part10.test.ts": 15799, + "apps/webapp/test/runsReplicationService.part2.test.ts": 24084, + "apps/webapp/test/runsReplicationService.part3.test.ts": 12625, + "apps/webapp/test/runsReplicationService.part4.test.ts": 31142, + "apps/webapp/test/runsReplicationService.part5.test.ts": 8925, + "apps/webapp/test/runsReplicationService.part6.test.ts": 13400, + "apps/webapp/test/runsReplicationService.part7.test.ts": 69637, + "apps/webapp/test/runsReplicationService.part8.test.ts": 28406, + "apps/webapp/test/runsReplicationService.part9.test.ts": 9244, + "apps/webapp/test/runsReplicationServiceExternalDeploymentId.test.ts": 8023, + "apps/webapp/test/runsRepository.part1.test.ts": 27264, + "apps/webapp/test/runsRepository.part2.test.ts": 23743, + "apps/webapp/test/runsRepository.part3.test.ts": 20003, + "apps/webapp/test/runsRepository.part4.test.ts": 24817, + "apps/webapp/test/runsRepository.readthrough.test.ts": 36287, + "apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts": 3779, + "apps/webapp/test/runsRepositoryCpres.test.ts": 7073, + "apps/webapp/test/runsRepositoryCursor.test.ts": 26167, + "apps/webapp/test/safeEnvironmentLog.test.ts": 2, + "apps/webapp/test/safeIntegrationLog.test.ts": 3, + "apps/webapp/test/safeRequestLogContext.test.ts": 16, + "apps/webapp/test/safeWebhookFetch.test.ts": 4, + "apps/webapp/test/safeWebhookUrl.test.ts": 6, + "apps/webapp/test/sameOriginNavigation.test.ts": 14, + "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 11, + "apps/webapp/test/sanitizeSessionInput.server.test.ts": 3, + "apps/webapp/test/sanitizeUrl.test.ts": 2, + "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 3, + "apps/webapp/test/scheduleTimings.test.ts": 2150, + "apps/webapp/test/scheduleWindow.test.ts": 27, + "apps/webapp/test/schedulesPutEnvScoping.test.ts": 10070, + "apps/webapp/test/selectBestEnvironment.test.ts": 4, + "apps/webapp/test/sentryRequestIsolation.test.ts": 73, + "apps/webapp/test/sentryTenantContext.test.ts": 3, + "apps/webapp/test/sentryTraceContext.server.test.ts": 6, + "apps/webapp/test/services.controlPlane.readthrough.test.ts": 4862, + "apps/webapp/test/services/organizationAccessToken.test.ts": 5, + "apps/webapp/test/services/personalAccessToken.test.ts": 6, + "apps/webapp/test/session-agent.e2e.test.ts": 73920, + "apps/webapp/test/session-stream.browser.e2e.test.ts": 20470, + "apps/webapp/test/session-stream.e2e.test.ts": 30404, + "apps/webapp/test/sessionDuration.test.ts": 11876, + "apps/webapp/test/sessionRunStreamsBackend.e2e.test.ts": 20771, + "apps/webapp/test/sessionRunStreamsPerOrgBasin.e2e.test.ts": 20639, + "apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts": 3529, + "apps/webapp/test/sessions.readthrough.test.ts": 6749, + "apps/webapp/test/sessionsReplicationService.test.ts": 17485, + "apps/webapp/test/setActiveOnTaskSchedule.test.ts": 9330, + "apps/webapp/test/shouldRevalidateRunsList.test.ts": 4, + "apps/webapp/test/slackErrorAlerts.test.ts": 1, + "apps/webapp/test/slackOAuthResultLog.test.ts": 2, + "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5779, + "apps/webapp/test/spanTraceRoutes.replicaLag.test.ts": 5917, + "apps/webapp/test/streamBatchItemsAuthorization.test.ts": 5, + "apps/webapp/test/streamLoader.controlPlane.test.ts": 4970, + "apps/webapp/test/syncDeclarativeSchedules.test.ts": 9761, + "apps/webapp/test/syncDeclarativeWebhooks.test.ts": 10183, + "apps/webapp/test/taskCodeSnippets.test.ts": 4, + "apps/webapp/test/tenantContext.test.ts": 24, + "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 2, + "apps/webapp/test/tenantContextResolver.test.ts": 15, + "apps/webapp/test/themePreference.test.ts": 4, + "apps/webapp/test/timeGranularity.test.ts": 4, + "apps/webapp/test/timelineSpanEvents.test.ts": 6, + "apps/webapp/test/traceExport.test.ts": 6, + "apps/webapp/test/uatEnvironmentClaim.test.ts": 32, + "apps/webapp/test/updateMetadata.test.ts": 20384, + "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 7987, + "apps/webapp/test/useTableSort.test.ts": 7, + "apps/webapp/test/userActorEnvironmentScopeRouteBuilder.test.ts": 28, + "apps/webapp/test/userActorPatOnlyBoundary.test.ts": 3254, + "apps/webapp/test/userActorProjectWideScope.test.ts": 3735, + "apps/webapp/test/userActorSourcePat.test.ts": 38, + "apps/webapp/test/userActorTokenClaimsAndScopes.test.ts": 3785, + "apps/webapp/test/utils/timezones.test.ts": 14, + "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 6795, + "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 8151, + "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 7306, + "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4339, + "apps/webapp/test/validateGitBranchName.test.ts": 4, + "apps/webapp/test/vercelUrls.test.ts": 5, + "apps/webapp/test/verifyDeploymentImage.test.ts": 612, + "apps/webapp/test/viewAsUser.test.ts": 9, + "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 7198, + "apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts": 3117, + "apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts": 5628, + "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 9896, + "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 5254, + "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 6177, + "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 4590, + "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 5203, + "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 30114, + "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5073, + "apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts": 7181, + "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 6260, + "apps/webapp/test/webhookErrorAlerts.test.ts": 4, + "apps/webapp/test/workerGroupAccess.test.ts": 2, + "apps/webapp/test/workerIdUnwrap.test.ts": 20, + "apps/webapp/test/workerQueueSplit.server.test.ts": 3, + "apps/webapp/test/workerQueueSplit.test.ts": 10, + "apps/webapp/test/workerRegions.test.ts": 5, + "apps/webapp/test/workloadTokenAuthorization.test.ts": 2, + "apps/webapp/test/workloadTokenGate.integration.test.ts": 2681, + "apps/webapp/test/writableEnvironments.test.ts": 3, "internal-packages/cache/src/stores/lruMemory.test.ts": 65, "internal-packages/clickhouse/src/client/client.test.ts": 7547, "internal-packages/clickhouse/src/taskRuns.test.ts": 6768,