c6f0769299
## Summary The logs search page (behind a feature flag) ran ClickHouse out of memory when browsing back over long time ranges. This keeps it within bounded memory and fixes a pagination bug that could skip or duplicate rows at a page boundary. ## Fix Memory: the list query reads in sort-key order, which opens one read stream per part in the window, and on object storage those per-part read buffers dominate peak memory, so it scaled with the number of parts scanned. Two changes bound it: - The logs ClickHouse client caps the per-part read buffers via new env-tunable settings. The object-storage-only setting is opt-in, so it is never sent to a ClickHouse version that lacks it. - Recent-first window narrowing: rows come back newest first, so the presenter probes the most recent window and only widens toward the full requested range when a page is short. A busy environment fills a page from a few recent parts instead of scanning the whole range; a quiet one still returns every row in a couple of cheap reads. Correctness: the keyset cursor ordered on (triggered_timestamp, trace_id), which is not unique because the spans of a trace share both, so rows at a tie could be skipped or duplicated across pages. The cursor and ORDER BY now include span_id, and the cursor is versioned so stale cursors reset to the first page. Guards: the effective page size is capped, and the existing per-query memory limit lets a pathological wide browse fail with an error instead of taking the node down. ## ClickHouse 26.2 The memory fix relies on lazy materialization deferring the wide attributes column to the output rows, which only holds on 26.x. Cloud already runs 26.2, so this moves the dev stack, testcontainers, and CI to match. The ClickHouse test suite passes on 26.2. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
161 lines
5.4 KiB
YAML
161 lines
5.4 KiB
YAML
name: "🧪 Unit Tests: Packages"
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
on:
|
|
workflow_call:
|
|
secrets:
|
|
DOCKERHUB_USERNAME:
|
|
required: false
|
|
DOCKERHUB_TOKEN:
|
|
required: false
|
|
|
|
jobs:
|
|
unitTests:
|
|
name: "🧪 Unit Tests: Packages"
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
# one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard
|
|
fail-fast: false
|
|
matrix:
|
|
shardIndex: [1, 2, 3]
|
|
shardTotal: [3]
|
|
env:
|
|
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
SHARD_INDEX: ${{ matrix.shardIndex }}
|
|
SHARD_TOTAL: ${{ matrix.shardTotal }}
|
|
steps:
|
|
- name: 🔧 Disable IPv6
|
|
run: |
|
|
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
|
|
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
|
|
sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
|
|
|
|
- name: 🔧 Configure docker address pool
|
|
run: |
|
|
CONFIG='{
|
|
"default-address-pools" : [
|
|
{
|
|
"base" : "172.17.0.0/12",
|
|
"size" : 20
|
|
},
|
|
{
|
|
"base" : "192.168.0.0/16",
|
|
"size" : 24
|
|
}
|
|
]
|
|
}'
|
|
mkdir -p /etc/docker
|
|
echo "$CONFIG" | sudo tee /etc/docker/daemon.json
|
|
|
|
- name: 🔧 Restart docker daemon
|
|
run: sudo systemctl restart docker
|
|
|
|
- name: ⬇️ Checkout repo
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: ⎔ Setup pnpm
|
|
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
|
with:
|
|
version: 10.33.2
|
|
|
|
- name: ⎔ Setup node
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: 20.20.2
|
|
cache: "pnpm"
|
|
|
|
# ..to avoid rate limits when pulling images
|
|
- name: 🐳 Login to DockerHub
|
|
if: ${{ env.DOCKERHUB_USERNAME }}
|
|
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
- name: 🐳 Skipping DockerHub login (no secrets available)
|
|
if: ${{ !env.DOCKERHUB_USERNAME }}
|
|
run: echo "DockerHub login skipped because secrets are not available."
|
|
|
|
- name: 🐳 Pre-pull testcontainer images
|
|
if: ${{ env.DOCKERHUB_USERNAME }}
|
|
run: |
|
|
# Retry each pull - DockerHub registry timeouts are a recurring transient CI flake.
|
|
pull() {
|
|
for attempt in 1 2 3; do
|
|
docker pull "$1" && return 0
|
|
echo "::warning::docker pull $1 failed (attempt ${attempt}/3); retrying in 10s"
|
|
sleep 10
|
|
done
|
|
echo "::error::docker pull $1 failed after 3 attempts"
|
|
return 1
|
|
}
|
|
echo "Pre-pulling Docker images with authenticated session..."
|
|
pull postgres:14
|
|
pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251
|
|
pull redis:7.2
|
|
pull testcontainers/ryuk:0.14.0
|
|
pull electricsql/electric:1.2.4
|
|
echo "Image pre-pull complete"
|
|
|
|
- name: 📥 Download deps
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: 📀 Generate Prisma Client
|
|
run: pnpm run generate
|
|
|
|
- name: 🧪 Run Package Unit Tests
|
|
run: pnpm run test:packages --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --passWithNoTests
|
|
|
|
- name: Gather all reports
|
|
if: ${{ !cancelled() }}
|
|
run: |
|
|
mkdir -p .vitest-reports
|
|
find . -type f -path '*/.vitest-reports/blob-*.json' \
|
|
-exec bash -c 'src="$1"; basename=$(basename "$src"); pkg=$(dirname "$src" | sed "s|^\./||;s|/\.vitest-reports$||;s|/|_|g"); cp "$src" ".vitest-reports/${pkg}-${basename}"' _ {} \;
|
|
|
|
- name: Upload blob reports to GitHub Actions Artifacts
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: packages-blob-report-${{ matrix.shardIndex }}
|
|
path: .vitest-reports/*
|
|
include-hidden-files: true
|
|
retention-days: 1
|
|
|
|
merge-reports:
|
|
name: "📊 Merge Reports"
|
|
if: ${{ !cancelled() }}
|
|
needs: [unitTests]
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: ⬇️ Checkout repo
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: ⎔ Setup pnpm
|
|
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
|
|
with:
|
|
version: 10.33.2
|
|
|
|
- name: ⎔ Setup node
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: 20.20.2
|
|
# no cache enabled, we're not installing deps
|
|
|
|
- name: Download blob reports from GitHub Actions Artifacts
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
path: .vitest-reports
|
|
pattern: packages-blob-report-*
|
|
merge-multiple: true
|
|
|
|
- name: Merge reports
|
|
run: pnpm dlx vitest@4.1.7 run --merge-reports --pass-with-no-tests
|