From 707bf1adb427ce20fb11ded31ddaf6b927d55928 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 5 Jun 2026 17:59:11 +0100 Subject: [PATCH] ci: reduce unit test flakiness and shard re-run cost (#3844) A unit-test shard recently failed on a timing race rather than a real regression - a run-engine waitpoint test sleeps 1250ms waiting on a 1000ms timeout that's processed by a ~1000ms worker poll, so on a CPU-starved shard the margin evaporates and the whole matrix goes red. Because `fail-fast` defaults on, that one flake cancels the sibling shards, and the only recovery is re-running the entire matrix "just to be sure" - which is itself slow. This is the low-risk first pass at that pain: - `fail-fast: false` on the webapp and internal shard matrices, so one flaky shard no longer cancels its siblings. "Re-run failed jobs" now re-runs just the failed shard instead of the whole matrix. - CI-scoped `retry: process.env.CI ? 2 : 0` on the timing-sensitive packages (`run-engine`, `redis-worker`, `schedule-engine`). Flakes self-heal in CI; local runs stay at `retry: 0` so they still surface in dev. A stopgap until the timing tests are made deterministic. - `fetch-depth: 1` on the unit-test checkouts - they don't use git history, so the full clone was wasted setup time across ~20 jobs. - Reconcile the pre-pull image tags with what testcontainers actually pulls (`redis:7-alpine` -> `redis:7.2`, `ryuk:0.11.0` -> `ryuk:0.14.0`) and add `minio/minio:latest` to the webapp pre-pull. Otherwise those images pull unauthenticated at test time and risk Docker Hub rate-limit flakes (worst on fork PRs, where the authenticated pre-pull is skipped entirely). Deeper follow-ups - bigger runners, turbo remote cache, runtime-weighted sharding, and the real root-cause fix (container reuse / template-DB isolation + deterministic timing tests) - are tracked under TRI-10484. --- .github/workflows/unit-tests-internal.yml | 10 ++++++---- .github/workflows/unit-tests-packages.yml | 8 ++++---- .github/workflows/unit-tests-webapp.yml | 11 +++++++---- internal-packages/run-engine/vitest.config.ts | 2 ++ internal-packages/schedule-engine/vitest.config.ts | 2 ++ packages/redis-worker/vitest.config.ts | 2 ++ 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/unit-tests-internal.yml b/.github/workflows/unit-tests-internal.yml index eb441856b..6f2b32f62 100644 --- a/.github/workflows/unit-tests-internal.yml +++ b/.github/workflows/unit-tests-internal.yml @@ -16,6 +16,8 @@ jobs: name: "🧪 Unit Tests: Internal" runs-on: ubuntu-latest strategy: + # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard + fail-fast: false matrix: shardIndex: [1, 2, 3, 4, 5, 6, 7, 8] shardTotal: [8] @@ -53,7 +55,7 @@ jobs: - name: ⬇️ Checkout repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: ⎔ Setup pnpm @@ -84,8 +86,8 @@ jobs: echo "Pre-pulling Docker images with authenticated session..." docker pull postgres:14 docker pull clickhouse/clickhouse-server:25.4-alpine - docker pull redis:7-alpine - docker pull testcontainers/ryuk:0.11.0 + docker pull redis:7.2 + docker pull testcontainers/ryuk:0.14.0 docker pull electricsql/electric:1.2.4 echo "Image pre-pull complete" @@ -123,7 +125,7 @@ jobs: - name: ⬇️ Checkout repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: ⎔ Setup pnpm diff --git a/.github/workflows/unit-tests-packages.yml b/.github/workflows/unit-tests-packages.yml index 6dca46819..5251a9933 100644 --- a/.github/workflows/unit-tests-packages.yml +++ b/.github/workflows/unit-tests-packages.yml @@ -53,7 +53,7 @@ jobs: - name: ⬇️ Checkout repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: ⎔ Setup pnpm @@ -84,8 +84,8 @@ jobs: echo "Pre-pulling Docker images with authenticated session..." docker pull postgres:14 docker pull clickhouse/clickhouse-server:25.4-alpine - docker pull redis:7-alpine - docker pull testcontainers/ryuk:0.11.0 + docker pull redis:7.2 + docker pull testcontainers/ryuk:0.14.0 docker pull electricsql/electric:1.2.4 echo "Image pre-pull complete" @@ -123,7 +123,7 @@ jobs: - name: ⬇️ Checkout repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: ⎔ Setup pnpm diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index 8c38e3253..3517afbba 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -16,6 +16,8 @@ jobs: name: "🧪 Unit Tests: Webapp" runs-on: ubuntu-latest strategy: + # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard + fail-fast: false matrix: shardIndex: [1, 2, 3, 4, 5, 6, 7, 8] shardTotal: [8] @@ -53,7 +55,7 @@ jobs: - name: ⬇️ Checkout repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: ⎔ Setup pnpm @@ -84,9 +86,10 @@ jobs: echo "Pre-pulling Docker images with authenticated session..." docker pull postgres:14 docker pull clickhouse/clickhouse-server:25.4-alpine - docker pull redis:7-alpine - docker pull testcontainers/ryuk:0.11.0 + docker pull redis:7.2 + docker pull testcontainers/ryuk:0.14.0 docker pull electricsql/electric:1.2.4 + docker pull minio/minio:latest echo "Image pre-pull complete" - name: 📥 Download deps @@ -131,7 +134,7 @@ jobs: - name: ⬇️ Checkout repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: ⎔ Setup pnpm diff --git a/internal-packages/run-engine/vitest.config.ts b/internal-packages/run-engine/vitest.config.ts index c00c54387..fc9a8f271 100644 --- a/internal-packages/run-engine/vitest.config.ts +++ b/internal-packages/run-engine/vitest.config.ts @@ -4,6 +4,8 @@ export default defineConfig({ test: { include: ["**/*.test.ts"], globals: true, + // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention + retry: process.env.CI ? 2 : 0, isolate: true, fileParallelism: false, testTimeout: 120_000, diff --git a/internal-packages/schedule-engine/vitest.config.ts b/internal-packages/schedule-engine/vitest.config.ts index cce9bda54..e773ec6d1 100644 --- a/internal-packages/schedule-engine/vitest.config.ts +++ b/internal-packages/schedule-engine/vitest.config.ts @@ -3,6 +3,8 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { globals: true, + // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention + retry: process.env.CI ? 2 : 0, environment: "node", setupFiles: ["./test/setup.ts"], testTimeout: 30000, diff --git a/packages/redis-worker/vitest.config.ts b/packages/redis-worker/vitest.config.ts index c4d431e10..452a66037 100644 --- a/packages/redis-worker/vitest.config.ts +++ b/packages/redis-worker/vitest.config.ts @@ -4,6 +4,8 @@ export default defineConfig({ test: { include: ["**/*.test.ts"], globals: true, + // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention + retry: process.env.CI ? 2 : 0, fileParallelism: false, }, });