update agents.md

Co-authored-by: Yuge Zhang <scottyugochang@gmail.com>

.dockerignore

@@ -0,0 +1,14 @@
+.venv
+**/.venv
+__pycache__
+.git
+.gitignore
+**/node_modules
+dist
+build
+.env
+docker
+.pytest_cache
+.vscode
+**/*.log
+examples/**/data

.github/workflows/badge-azure.yml

@@ -0,0 +1,29 @@
+name: Badge - Azure
+
+on:
+  workflow_run:
+    workflows:
+      - Examples - Azure
+    types: [completed]
+
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  badge:
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'main') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const badgeAggregation = require('./scripts/badge_aggregation.js');
+            const dependencies = [
+              { workflow: 'examples-azure.yml', label: 'azure', variants: ['stable'] },
+            ];
+            await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-claude-code.yml

@@ -0,0 +1,29 @@
+name: Badge - Claude Code
+
+on:
+  workflow_run:
+    workflows:
+      - Examples - Claude Code
+    types: [completed]
+
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  badge:
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'main') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const badgeAggregation = require('./scripts/badge_aggregation.js');
+            const dependencies = [
+              { workflow: 'examples-claude-code.yml', label: 'claude-code', variants: ['stable'] },
+            ];
+            await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-compat.yml

@@ -0,0 +1,29 @@
+name: Badge - Compatibility
+
+on:
+  workflow_run:
+    workflows:
+      - Examples - Backward Compatibility
+    types: [completed]
+
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  badge:
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'main') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const badgeAggregation = require('./scripts/badge_aggregation.js');
+            const dependencies = [
+              { workflow: 'examples-compat.yml', label: 'examples-compat', variants: ['legacy', 'stable'] },
+            ];
+            await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-examples.yml

@@ -7,6 +7,10 @@ on:
       - Examples - Spider
       - Examples - APO
       - Examples - Unsloth
+      - Examples - Tinker
+      - Examples - Azure
+      - Examples - Claude Code
+      - Examples - RAG
     types: [completed]
 
   workflow_dispatch:
@@ -31,5 +35,9 @@ jobs:
               { workflow: 'examples-spider.yml', label: 'examples-spider.stable', variants: ['stable'] },
               { workflow: 'examples-apo.yml', label: 'examples-apo.stable', variants: ['stable'] },
               { workflow: 'examples-unsloth.yml', label: 'examples-unsloth.stable', variants: ['stable'] },
+              { workflow: 'examples-tinker.yml', label: 'examples-tinker.stable', variants: ['stable'] },
+              { workflow: 'examples-azure.yml', label: 'examples-azure.stable', variants: ['stable'] },
+              { workflow: 'examples-claude-code.yml', label: 'examples-claude-code.stable', variants: ['stable'] },
+              { workflow: 'examples-rag.yml', label: 'examples-rag.stable', variants: ['stable'] },
             ];
             await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-latest.yml

@@ -7,6 +7,8 @@ on:
       - Examples - Spider
       - Examples - APO
       - Examples - Unsloth
+      - Examples - RAG
+      - Examples - Claude Code
       - GPU Test
     types: [completed]
 
@@ -32,6 +34,8 @@ jobs:
               { workflow: 'examples-spider.yml', label: 'spider.latest', variants: ['latest'] },
               { workflow: 'examples-apo.yml', label: 'apo.latest', variants: ['latest'] },
               { workflow: 'examples-unsloth.yml', label: 'unsloth.latest', variants: ['latest'] },
+              { workflow: 'examples-claude-code.yml', label: 'claude-code.latest', variants: ['latest'] },
+              { workflow: 'examples-rag.yml', label: 'rag.latest', variants: ['latest'] },
               { workflow: 'tests-full.yml', label: 'tests-full.latest', variants: ['latest'] },
             ];
             await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-rag.yml

@@ -0,0 +1,29 @@
+name: Badge - RAG
+
+on:
+  workflow_run:
+    workflows:
+      - Examples - RAG
+    types: [completed]
+
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  badge:
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'main') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const badgeAggregation = require('./scripts/badge_aggregation.js');
+            const dependencies = [
+              { workflow: 'examples-rag.yml', label: 'rag', variants: ['legacy', 'stable'] },
+            ];
+            await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-spider.yml

@@ -24,6 +24,6 @@ jobs:
           script: |
             const badgeAggregation = require('./scripts/badge_aggregation.js');
             const dependencies = [
-              { workflow: 'examples-spider.yml', label: 'spider', variants: ['stable', 'legacy'] },
+              { workflow: 'examples-spider.yml', label: 'spider', variants: ['stable'] },
             ];
             await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-tinker.yml

@@ -0,0 +1,29 @@
+name: Badge - Tinker
+
+on:
+  workflow_run:
+    workflows:
+      - Examples - Tinker
+    types: [completed]
+
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  badge:
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'main') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const badgeAggregation = require('./scripts/badge_aggregation.js');
+            const dependencies = [
+              { workflow: 'examples-tinker.yml', label: 'tinker', variants: ['stable'] },
+            ];
+            await badgeAggregation({ github, context, core, dependencies });

.github/workflows/badge-unit.yml

@@ -0,0 +1,31 @@
+name: Badge - Unit Test
+
+on:
+  workflow_run:
+    workflows:
+      - CPU Test
+      - GPU Test
+    types: [completed]
+
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  badge:
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'main') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const badgeAggregation = require('./scripts/badge_aggregation.js');
+            const dependencies = [
+              { workflow: 'tests-full.yml', label: 'tests-full', variants: ['legacy', 'stable'] },
+              { workflow: 'tests.yml', label: 'tests', variants: ['legacy', 'stable', 'Lint', 'documentation', 'JavaScript'] },
+            ];
+            await badgeAggregation({ github, context, core, dependencies });

.github/workflows/benchmark.yml

@@ -0,0 +1,334 @@
+name: Benchmark
+permissions:
+  contents: read
+on:
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    name: Benchmark (${{ matrix.backend.id }}, ${{ matrix.scenario.display }})
+    runs-on: [self-hosted, 1ES.Pool=agl-runner-cpu]
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        backend:
+          - id: memory
+            compose_file: compose.prometheus-memory-store.yml
+          - id: mongo
+            compose_file: compose.prometheus-mongo-store.yml
+        scenario:
+          - id: minimal-production
+            display: Minimal production scale
+            store_workers: 4
+            args: >-
+              --mode batch
+              --total-tasks 4096
+              --batch-size 256
+              --n-runners 32
+              --max-rounds 6
+              --sleep-seconds 0.5
+          - id: medium-production
+            display: Medium production scale
+            store_workers: 16
+            args: >-
+              --mode batch
+              --total-tasks 10000
+              --batch-size 1000
+              --n-runners 100
+              --max-rounds 10
+              --sleep-seconds 0.1
+          - id: large-batch
+            display: Large batch waves
+            store_workers: 32
+            args: >-
+              --mode batch
+              --total-tasks 100000
+              --batch-size 8192
+              --n-runners 256
+              --max-rounds 6
+              --sleep-seconds 0.1
+          - id: long-queues
+            display: Long rollout queues
+            store_workers: 32
+            args: >-
+              --mode batch_partial
+              --total-tasks 100000
+              --batch-size 1024
+              --n-runners 256
+              --remaining-tasks 4096
+              --max-rounds 4
+              --sleep-seconds 0.1
+          - id: high-concurrency
+            display: High-throughput concurrent requests
+            store_workers: 32
+            args: >-
+              --mode single
+              --total-tasks 100000
+              --concurrency 2048
+              --n-runners 256
+              --max-rounds 2
+              --sleep-seconds 0.1
+          - id: heavy-traces
+            display: Heavy rollouts with deep traces
+            store_workers: 64
+            args: >-
+              --mode batch_partial
+              --total-tasks 10000
+              --batch-size 1024
+              --remaining-tasks 256
+              --n-runners 512
+              --max-rounds 20
+              --sleep-seconds 1.0
+    env:
+      STORE_URL: http://localhost:4747
+      STORE_API_URL: http://localhost:4747/v1/agl
+      PROM_URL: http://localhost:9090
+      SCENARIO_ID: ${{ matrix.scenario.id }}
+      BACKEND_ID: ${{ matrix.backend.id }}
+      ARTIFACT_DIR: artifacts/${{ matrix.scenario.id }}-${{ matrix.backend.id }}
+      COMPOSE_FILE: ${{ matrix.backend.compose_file }}
+      AGL_STORE_N_WORKERS: ${{ matrix.scenario.store_workers }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: '3.12'
+
+      - name: Sync dependencies
+        run: uv sync --frozen --extra mongo --group core-stable --group dev
+
+      - name: Check disk space
+        run: df -h
+
+      - name: Reset benchmark data directories
+        run: |
+          set -euo pipefail
+          cd docker
+          rm -rf data
+          bash setup.sh
+
+      - name: Launch ${{ matrix.backend.id }} Prometheus stack
+        run: |
+          set -euo pipefail
+          cd docker
+          docker compose -f "$COMPOSE_FILE" down -v || true
+          docker compose -f "$COMPOSE_FILE" up -d --quiet-pull
+
+      - name: Wait for store readiness
+        run: |
+          set -euo pipefail
+          for attempt in {1..60}; do
+            if curl -fsS "$STORE_API_URL/health" >/dev/null 2>&1; then
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Store did not become ready in time" >&2
+          docker compose -f "$COMPOSE_FILE" logs app
+          exit 1
+
+      - name: Prepare artifact directory
+        run: mkdir -p "$ARTIFACT_DIR"
+
+      - name: Record benchmark start
+        run: echo "BENCHMARK_START=$(date -u +%FT%TZ)" >> "$GITHUB_ENV"
+
+      - name: Run ${{ matrix.scenario.display }} workload
+        run: |
+          set -euo pipefail
+          uv run --locked --no-sync python -m tests.benchmark.benchmark_store \
+            --store-url "$STORE_URL" \
+            ${{ matrix.scenario.args }}
+
+      - name: Record benchmark end
+        if: ${{ always() }}
+        run: echo "BENCHMARK_END=$(date -u +%FT%TZ)" >> "$GITHUB_ENV"
+
+      - name: Run benchmark analysis
+        if: ${{ always() }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$ARTIFACT_DIR"
+          if [ -z "${BENCHMARK_START:-}" ] || [ -z "${BENCHMARK_END:-}" ]; then
+            echo "Analysis skipped: benchmark window not recorded." > "$ARTIFACT_DIR/analysis.txt"
+            exit 1
+          fi
+          uv run --locked --no-sync python -m tests.benchmark.analysis \
+            --prom-url "$PROM_URL" \
+            --store-url "$STORE_API_URL" \
+            --start "$BENCHMARK_START" \
+            --end "$BENCHMARK_END" \
+            | tee "$ARTIFACT_DIR/analysis.txt"
+
+      - name: Stop ${{ matrix.backend.id }} Prometheus stack
+        if: ${{ always() }}
+        run: |
+          set -euo pipefail
+          cd docker
+          docker compose -f "$COMPOSE_FILE" down -v || true
+
+      - name: Archive Prometheus metrics
+        if: ${{ always() }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$ARTIFACT_DIR"
+          if [ -d docker/data/prometheus ]; then
+            tar -C docker/data -czf "$ARTIFACT_DIR/prometheus-${SCENARIO_ID}-${BACKEND_ID}.tar.gz" prometheus
+          fi
+          if docker compose -f "$COMPOSE_FILE" ps --format '{{.Name}}' >/dev/null 2>&1; then
+            docker compose -f "$COMPOSE_FILE" logs app > "$ARTIFACT_DIR/docker-${SCENARIO_ID}-${BACKEND_ID}.log" || true
+          fi
+
+      - name: Upload benchmark artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-${{ matrix.scenario.id }}-${{ matrix.backend.id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: error
+
+  micro-benchmark:
+    name: Micro-benchmark (${{ matrix.backend.id }}, ${{ matrix.mode.display }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        backend:
+          - id: memory
+            compose_file: compose.prometheus-memory-store.yml
+          - id: mongo
+            compose_file: compose.prometheus-mongo-store.yml
+        mode:
+          - id: worker
+            display: Update worker throughput
+            cli: worker
+          - id: dequeue-empty
+            display: Dequeue empty throughput
+            cli: dequeue-empty
+          - id: rollout
+            display: Rollout + span throughput
+            cli: rollout
+    env:
+      STORE_URL: http://localhost:4747
+      STORE_API_URL: http://localhost:4747/v1/agl
+      PROM_URL: http://localhost:9090
+      BACKEND_ID: ${{ matrix.backend.id }}
+      MODE_ID: ${{ matrix.mode.id }}
+      ARTIFACT_DIR: artifacts/micro-${{ matrix.mode.id }}-${{ matrix.backend.id }}
+      COMPOSE_FILE: ${{ matrix.backend.compose_file }}
+      AGL_STORE_N_WORKERS: 8
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: '3.12'
+
+      - name: Sync dependencies
+        run: uv sync --frozen --extra mongo --group core-stable --group dev
+
+      - name: Reset benchmark data directories
+        run: |
+          set -euo pipefail
+          cd docker
+          rm -rf data
+          bash setup.sh
+
+      - name: Launch ${{ matrix.backend.id }} Prometheus stack
+        run: |
+          set -euo pipefail
+          cd docker
+          docker compose -f "$COMPOSE_FILE" down -v || true
+          docker compose -f "$COMPOSE_FILE" up -d --quiet-pull
+
+      - name: Wait for store readiness
+        run: |
+          set -euo pipefail
+          for attempt in {1..60}; do
+            if curl -fsS "$STORE_API_URL/health" >/dev/null 2>&1; then
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Store did not become ready in time" >&2
+          cd docker && docker compose -f "$COMPOSE_FILE" logs app
+          exit 1
+
+      - name: Prepare artifact directory
+        run: mkdir -p "$ARTIFACT_DIR"
+
+      - name: Record micro benchmark start
+        run: echo "BENCHMARK_START=$(date -u +%FT%TZ)" >> "$GITHUB_ENV"
+
+      - name: Run ${{ matrix.mode.display }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$ARTIFACT_DIR"
+          uv run --locked --no-sync python -m tests.benchmark.micro_benchmark \
+            --store-url "$STORE_URL" \
+            --summary-file "$ARTIFACT_DIR/summary-${MODE_ID}.txt" \
+            "${{ matrix.mode.cli }}" | tee "$ARTIFACT_DIR/micro-${MODE_ID}.txt"
+
+      - name: Record micro benchmark end
+        if: ${{ always() }}
+        run: echo "BENCHMARK_END=$(date -u +%FT%TZ)" >> "$GITHUB_ENV"
+
+      - name: Run micro benchmark analysis
+        if: ${{ always() }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$ARTIFACT_DIR"
+          if [ -z "${BENCHMARK_START:-}" ] || [ -z "${BENCHMARK_END:-}" ]; then
+            echo "Analysis skipped: benchmark window not recorded." > "$ARTIFACT_DIR/analysis-${MODE_ID}.txt"
+            exit 1
+          fi
+          uv run --locked --no-sync python -m tests.benchmark.analysis \
+            --prom-url "$PROM_URL" \
+            --store-url "$STORE_API_URL" \
+            --start "$BENCHMARK_START" \
+            --end "$BENCHMARK_END" \
+            | tee "$ARTIFACT_DIR/analysis-${MODE_ID}.txt"
+
+      - name: Show micro benchmark summary
+        if: ${{ always() }}
+        run: |
+          set -euo pipefail
+          summary_file="$ARTIFACT_DIR/summary-${MODE_ID}.txt"
+          if [ -f "$summary_file" ]; then
+            echo "Micro benchmark summary ($MODE_ID/$BACKEND_ID):"
+            cat "$summary_file"
+          else
+            echo "Summary file not found: $summary_file"
+          fi
+
+      - name: Stop ${{ matrix.backend.id }} Prometheus stack
+        if: ${{ always() }}
+        run: |
+          set -euo pipefail
+          cd docker
+          docker compose -f "$COMPOSE_FILE" down -v || true
+
+      - name: Archive Prometheus metrics
+        if: ${{ always() }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$ARTIFACT_DIR"
+          if [ -d docker/data/prometheus ]; then
+            tar -C docker/data -czf "$ARTIFACT_DIR/prometheus-micro-${MODE_ID}-${BACKEND_ID}.tar.gz" prometheus
+          fi
+          if docker compose -f "$COMPOSE_FILE" ps --format '{{.Name}}' >/dev/null 2>&1; then
+            docker compose -f "$COMPOSE_FILE" logs app > "$ARTIFACT_DIR/docker-micro-${MODE_ID}-${BACKEND_ID}.log" || true
+          fi
+
+      - name: Upload micro benchmark artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: micro-benchmark-${{ matrix.mode.id }}-${{ matrix.backend.id }}
+          path: ${{ env.ARTIFACT_DIR }}
+          if-no-files-found: error

.github/workflows/examples-apo.yml

@@ -14,7 +14,7 @@ on:
 run-name: >-
   ${{ github.event_name == 'repository_dispatch'
       && format(
-        'PR #{0} - Label {1} - {2}',
+        'APO - PR #{0} - {1} - {2}',
         github.event.client_payload.pull_number,
         github.event.client_payload.ci_label,
         github.event.client_payload.correlation_id

.github/workflows/examples-azure.yml

@@ -0,0 +1,98 @@
+name: Examples - Azure
+permissions:
+  contents: read
+on:
+  schedule:
+    # Every day at 4 AM UTC+8
+    - cron: '0 20 * * *'
+
+  workflow_dispatch:
+
+  repository_dispatch:
+    types: [ci-azure, ci-all]
+
+run-name: >-
+  ${{ github.event_name == 'repository_dispatch'
+      && format(
+        'Azure - PR #{0} - {1} - {2}',
+        github.event.client_payload.pull_number,
+        github.event.client_payload.ci_label,
+        github.event.client_payload.correlation_id
+      )
+      || format('Azure - {0}', github.event_name) }}
+
+jobs:
+  azure:
+    if: >
+      github.event_name != 'repository_dispatch' ||
+      github.event.action == 'ci-azure' ||
+      github.event.action == 'ci-all'
+    name: Azure (Python ${{ matrix.python-version }}, ${{ matrix.setup-script }})
+    runs-on: [self-hosted, 1ES.Pool=agl-runner-cpu]
+    timeout-minutes: 400
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.12'
+          setup-script: 'stable'
+      fail-fast: false
+    steps:
+      - name: Check disk space
+        run: df -h
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_ref || (github.event.pull_request.number && format('refs/pull/{0}/merge', github.event.pull_request.number)) || github.ref }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade dependencies (latest)
+        run: uv lock --upgrade
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies
+        run: |
+          uv sync --frozen --no-default-groups \
+            --group dev --group experiment --group agents --group core-stable
+      - name: Freeze dependencies
+        run: |
+          set -ex
+          uv pip freeze | tee requirements-freeze.txt
+          echo "UV_LOCKED=1" >> $GITHUB_ENV
+          echo "UV_NO_SYNC=1" >> $GITHUB_ENV
+      - name: Upload dependencies artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dependencies-azure-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          path: requirements-freeze.txt
+          compression-level: 0
+
+      - name: Azure Login
+        run: |
+          az login --identity
+        shell: bash
+
+      - name: Azure OpenAI Sanity Check
+        run: |
+          source .venv/bin/activate
+          cd examples/azure
+          python capital_agent.py
+        shell: bash
+        env:
+          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT_SWEDEN }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_SWEDEN }}
+        id: azure_openai_sanity_check
+
+      - name: Azure OpenAI Supervised Fine-tuning
+        run: |
+          source .venv/bin/activate
+          cd examples/azure
+          python train_capital_agent.py --n-iterations 2 --cleanup
+        shell: bash
+        env:
+          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT_SWEDEN }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_SWEDEN }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          AZURE_OPENAI_API_VERSION: 2025-04-01-preview
+          AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }}
+          AZURE_RESOURCE_NAME: ${{ secrets.AZURE_RESOURCE_NAME }}
+        id: azure_openai_finetune

.github/workflows/examples-calc-x.yml

@@ -14,7 +14,7 @@ on:
 run-name: >-
   ${{ github.event_name == 'repository_dispatch'
       && format(
-        'PR #{0} - Label {1} - {2}',
+        'Calc-X - PR #{0} - {1} - {2}',
         github.event.client_payload.pull_number,
         github.event.client_payload.ci_label,
         github.event.client_payload.correlation_id
@@ -22,12 +22,12 @@ run-name: >-
       || format('Calc-X - {0}', github.event_name) }}
 
 jobs:
-  calc-x:
+  calc-x-perf:
     if: >
       github.event_name != 'repository_dispatch' ||
       github.event.action == 'ci-calc-x' ||
       github.event.action == 'ci-all'
-    name: Calc-X (Python ${{ matrix.python-version }}, ${{ matrix.setup-script }})
+    name: Calc-X Performance (Python ${{ matrix.python-version }}, ${{ matrix.setup-script }})
     runs-on: [self-hosted, 1ES.Pool=agl-runner-gpu]
     timeout-minutes: 90
     strategy:
@@ -74,7 +74,7 @@ jobs:
       - name: Upload dependencies artifact
         uses: actions/upload-artifact@v4
         with:
-          name: dependencies-calc-x-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          name: dependencies-calc-x-performance-${{ matrix.python-version }}-${{ matrix.setup-script }}
           path: requirements-freeze.txt
           compression-level: 0
 
@@ -116,13 +116,11 @@ jobs:
       # Don't ask why. Don't touch this.
       - name: Calc-X training
         run: |
-          set -ex
           source .venv/bin/activate
           cd examples/calc_x
           ../../scripts/restart_ray.sh
           sleep 5
-          PYTHONUNBUFFERED=1 python train_calc_agent.py --val-file data/test_mini.parquet --ci
-          sleep 10
+          python train_calc_agent.py --val-file data/test_mini.parquet --ci
         shell: bash
         env:
           WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
@@ -137,14 +135,126 @@ jobs:
           WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
           WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
 
-      - name: Calc-X training LLM Proxy
+  calc-x-variants:
+    if: >
+      github.event_name != 'repository_dispatch' ||
+      github.event.action == 'ci-calc-x' ||
+      github.event.action == 'ci-all'
+    name: Calc-X Variants (Python ${{ matrix.python-version }}, ${{ matrix.setup-script }})
+    runs-on: [self-hosted, 1ES.Pool=agl-runner-gpu]
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.10'
+          setup-script: 'legacy'
+        - python-version: '3.12'
+          setup-script: 'stable'
+        - python-version: '3.13'
+          setup-script: 'latest'
+      fail-fast: false
+    steps:
+      - name: Check GPU status
+        run: nvidia-smi
+      - name: Check disk space
+        run: df -h
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_ref || (github.event.pull_request.number && format('refs/pull/{0}/merge', github.event.pull_request.number)) || github.ref }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade dependencies (latest)
+        run: uv lock --upgrade
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies (latest)
+        run: |
+          uv sync --frozen --no-default-groups --extra verl \
+            --group dev --group experiment --group agents --group torch-gpu-stable
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies (stable & legacy)
+        run: |
+          uv sync --frozen --no-default-groups --extra verl \
+            --group dev --group experiment --group agents --group torch-gpu-${{ matrix.setup-script }}
+        if: matrix.setup-script != 'latest'
+      - name: Freeze dependencies
+        run: |
+          set -ex
+          uv pip freeze | tee requirements-freeze.txt
+          echo "UV_LOCKED=1" >> $GITHUB_ENV
+          echo "UV_NO_SYNC=1" >> $GITHUB_ENV
+      - name: Upload dependencies artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dependencies-calc-x-variants-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          path: requirements-freeze.txt
+          compression-level: 0
+
+      - name: Launch LiteLLM Proxy
+        run: |
+          ./scripts/litellm_run.sh
+        env:
+          AZURE_API_BASE: ${{ secrets.AZURE_GROUP_SUBSCRIPTION_API_BASE }}
+          AZURE_API_KEY: ${{ secrets.AZURE_GROUP_SUBSCRIPTION_API_KEY }}
+
+      - name: Prepare Calc-X dataset
+        run: |
+          set -ex
+          cd examples/calc_x
+          uv run gdown --fuzzy https://drive.google.com/file/d/1FQMyKLLd6hP9dw9rfZn1EZOWNvKaDsqw/view
+          unzip calc-x-data.zip -d data
+          rm calc-x-data.zip
+
+      - name: Calc-X MCP sanity check
+        run: |
+          set -ex
+          cd examples/calc_x
+          uv run tests/test_mcp_calculator.py
+        env:
+          OPENAI_API_BASE: http://localhost:12306/
+          OPENAI_API_KEY: dummy
+      - name: Calc-X sanity check
+        run: |
+          set -ex
+          cd examples/calc_x
+          uv run legacy_calc_agent_debug.py
+        env:
+          OPENAI_BASE_URL: http://localhost:12306/
+          OPENAI_API_KEY: dummy
+
+      - name: Training with local model
         run: |
           set -ex
           source .venv/bin/activate
           cd examples/calc_x
           ../../scripts/restart_ray.sh
           sleep 5
-          PYTHONUNBUFFERED=1 python train_calc_agent.py --val-file data/test_mini.parquet --ci --llm-proxy
+          hf download Qwen/Qwen2.5-0.5B-Instruct --local-dir data/qwen_model
+          PYTHONUNBUFFERED=1 python train_calc_agent.py --val-file data/test_mini.parquet --ci-fast --model $(realpath data/qwen_model)
+          sleep 10
+        shell: bash
+        env:
+          WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
+          WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
+        id: calc_x_train_local_model
+
+      - name: Validate training with local model
+        run: |
+          set -ex
+          uv run scripts/validate_example_wandb.py ${{ steps.calc_x_train_local_model.outputs.project_name }} ${{ steps.calc_x_train_local_model.outputs.run_name }}
+        env:
+          WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
+          WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
+
+      - name: Training with LLM Proxy
+        run: |
+          set -ex
+          source .venv/bin/activate
+          cd examples/calc_x
+          ../../scripts/restart_ray.sh
+          sleep 5
+          PYTHONUNBUFFERED=1 python train_calc_agent.py --val-file data/test_mini.parquet --ci-fast --llm-proxy
           sleep 10
         shell: bash
         env:
@@ -152,7 +262,15 @@ jobs:
           WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
         id: calc_x_train_llm_proxy
 
-      - name: Calc-X training with external store
+      - name: Validate training with LLM Proxy
+        run: |
+          set -ex
+          uv run scripts/validate_example_wandb.py ${{ steps.calc_x_train_llm_proxy.outputs.project_name }} ${{ steps.calc_x_train_llm_proxy.outputs.run_name }}
+        env:
+          WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
+          WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
+
+      - name: Training with external store
         run: |
           set -euo pipefail
           source .venv/bin/activate
@@ -182,7 +300,15 @@ jobs:
           WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
         id: calc_x_train_external_store
 
-      - name: Calc-X training with role-based environment variables
+      - name: Validate training with external store
+        run: |
+          set -ex
+          uv run scripts/validate_example_wandb.py ${{ steps.calc_x_train_external_store.outputs.project_name }} ${{ steps.calc_x_train_external_store.outputs.run_name }}
+        env:
+          WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
+          WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
+
+      - name: Training with role-based environment variables
         run: |
           set -euo pipefail
           source .venv/bin/activate
@@ -203,3 +329,12 @@ jobs:
         env:
           WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
           WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
+        id: calc_x_train_role_based_env_var
+
+      - name: Validate training with role-based environment variables
+        run: |
+          set -ex
+          uv run scripts/validate_example_wandb.py ${{ steps.calc_x_train_role_based_env_var.outputs.project_name }} ${{ steps.calc_x_train_role_based_env_var.outputs.run_name }}
+        env:
+          WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
+          WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}

.github/workflows/examples-claude-code.yml

@@ -0,0 +1,151 @@
+name: Examples - Claude Code
+permissions:
+  contents: read
+on:
+  schedule:
+    # Every day at 4 AM UTC+8
+    - cron: "0 20 * * *"
+
+  workflow_dispatch:
+
+  repository_dispatch:
+    types: [ci-claude-code, ci-all]
+
+run-name: >-
+  ${{ github.event_name == 'repository_dispatch'
+      && format(
+        'Claude Code - PR #{0} - {1} - {2}',
+        github.event.client_payload.pull_number,
+        github.event.client_payload.ci_label,
+        github.event.client_payload.correlation_id
+      )
+      || format('Claude Code - {0}', github.event_name) }}
+
+jobs:
+  claude-code:
+    if: >
+      github.event_name != 'repository_dispatch' ||
+      github.event.action == 'ci-claude-code' ||
+      github.event.action == 'ci-all'
+    name: Claude Code (Python ${{ matrix.python-version }}, ${{ matrix.setup-script }})
+    runs-on: [self-hosted, 1ES.Pool=agl-runner-gpu]
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.12"
+            setup-script: "stable"
+          - python-version: "3.13"
+            setup-script: "latest"
+      fail-fast: false
+    steps:
+      - name: Check GPU status
+        run: nvidia-smi
+      - name: Check disk space
+        run: df -h
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_ref || (github.event.pull_request.number && format('refs/pull/{0}/merge', github.event.pull_request.number)) || github.ref }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade dependencies (latest)
+        run: uv lock --upgrade
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies
+        run: |
+          uv sync --frozen --no-default-groups \
+            --group dev --group experiment --group agents --group torch-gpu-stable
+      - name: Freeze dependencies
+        run: |
+          set -ex
+          uv pip freeze | tee requirements-freeze.txt
+          echo "UV_LOCKED=1" >> $GITHUB_ENV
+          echo "UV_NO_SYNC=1" >> $GITHUB_ENV
+      - name: Upload dependencies artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dependencies-claude-code-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          path: requirements-freeze.txt
+          compression-level: 0
+
+      - name: Download model
+        run: |
+          source .venv/bin/activate
+          python -c "from transformers import AutoModelForCausalLM; AutoModelForCausalLM.from_pretrained('Qwen/Qwen3-Coder-30B-A3B-Instruct')"
+
+      - name: Launch vLLM server
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          vllm serve Qwen/Qwen3-Coder-30B-A3B-Instruct \
+            --max-model-len 131072 \
+            --enable-auto-tool-choice \
+            --tool-call-parser qwen3_coder \
+            --port 45993 &
+
+          VLLM_READY=0
+          for i in {1..100}; do
+            if curl -sSf http://localhost:45993/v1/models > /dev/null 2>&1; then
+              echo "vLLM server is ready!"
+              VLLM_READY=1
+              break
+            fi
+            echo "Waiting for vLLM server to be ready... (${i})"
+            sleep 5
+          done
+          if [[ "$VLLM_READY" != "1" ]]; then
+            echo "vLLM server failed to start!"
+            exit 1
+          fi
+
+      - name: Claude Code sanity check with vLLM models
+        run: |
+          source .venv/bin/activate
+          cd examples/claude_code
+          python claude_code_agent.py vllm --backend-model-high Qwen/Qwen3-Coder-30B-A3B-Instruct --backend-model-low Qwen/Qwen3-Coder-30B-A3B-Instruct --base-url http://localhost:45993/v1 --debug
+        shell: bash
+
+      - name: Upload sanity check artifacts for vLLM
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: claude-code-sanity-check-vllm-${{ matrix.setup-script }}
+          path: |
+            examples/claude_code/data/
+            examples/claude_code/logs/
+          if-no-files-found: error
+
+      - name: Cleanup vLLM
+        run: |
+          set -euo pipefail
+          pkill -f vllm
+          for i in {1..60}; do
+            if ! pgrep -f vllm; then
+              break
+            fi
+            sleep 5
+          done
+          rm -rf examples/claude_code/data/
+          rm -rf examples/claude_code/logs/
+
+      - name: Claude Code sanity check with OpenAI models
+        run: |
+          source .venv/bin/activate
+          cd examples/claude_code
+          python claude_code_agent.py openai --backend-model-high gpt-5.1-codex-mini --backend-model-low gpt-4.1-mini --debug
+        shell: bash
+        env:
+          OPENAI_BASE_URL: ${{ secrets.AZURE_OPENAI_ENDPOINT_SWEDEN }}
+          OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_SWEDEN }}
+
+      - name: Upload sanity check artifacts for OpenAI
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: claude-code-sanity-check-openai-${{ matrix.setup-script }}
+          path: |
+            examples/claude_code/data/
+            examples/claude_code/logs/
+          if-no-files-found: error

.github/workflows/examples-compat.yml

@@ -14,7 +14,7 @@ on:
 run-name: >-
   ${{ github.event_name == 'repository_dispatch'
       && format(
-        'PR #{0} - Label {1} - {2}',
+        'Backward Compatibility - PR #{0} - {1} - {2}',
         github.event.client_payload.pull_number,
         github.event.client_payload.ci_label,
         github.event.client_payload.correlation_id
@@ -56,7 +56,7 @@ jobs:
             --group dev --group experiment --group agents --group torch-gpu-${{ matrix.setup-script }}
       - name: Override VERL (stable)
         run: |
-          uv pip install verl==0.5.0
+          uv pip install verl==0.5.0 vllm==0.10.2
         if: matrix.setup-script == 'stable'
       - name: Freeze dependencies
         run: |

.github/workflows/examples-rag.yml

@@ -0,0 +1,179 @@
+name: Examples - RAG
+permissions:
+  contents: read
+on:
+  schedule:
+    # Every day at 6 AM UTC+8
+    - cron: '0 22 * * *'
+
+  workflow_dispatch:
+
+  repository_dispatch:
+    types: [ci-rag, ci-all]
+
+run-name: >-
+  ${{ github.event_name == 'repository_dispatch'
+      && format(
+        'RAG - PR #{0} - {1} - {2}',
+        github.event.client_payload.pull_number,
+        github.event.client_payload.ci_label,
+        github.event.client_payload.correlation_id
+      )
+      || format('RAG - {0}', github.event_name) }}
+
+jobs:
+  rag:
+    if: >
+      github.event_name != 'repository_dispatch' ||
+      github.event.action == 'ci-rag' ||
+      github.event.action == 'ci-all'
+    name: RAG (Python ${{ matrix.python-version }}, ${{ matrix.setup-script }})
+    runs-on: [self-hosted, 1ES.Pool=agl-runner-gpu]
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.10'
+          setup-script: 'legacy'
+        - python-version: '3.12'
+          setup-script: 'stable'
+        - python-version: '3.13'
+          setup-script: 'latest'
+      fail-fast: false
+    steps:
+      - name: Check GPU status
+        run: nvidia-smi
+      - name: Check disk space
+        run: df -h
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_ref || (github.event.pull_request.number && format('refs/pull/{0}/merge', github.event.pull_request.number)) || github.ref }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade dependencies (latest)
+        run: uv lock --upgrade
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies (latest)
+        run: |
+          uv sync --frozen --no-default-groups --extra verl \
+            --group dev --group experiment --group agents --group rag --group torch-gpu-stable
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies (stable & legacy)
+        run: |
+          uv sync --frozen --no-default-groups --extra verl \
+            --group dev --group experiment --group agents --group rag --group torch-gpu-${{ matrix.setup-script }}
+        if: matrix.setup-script != 'latest'
+      - name: Freeze dependencies
+        run: |
+          set -ex
+          uv pip freeze | tee requirements-freeze.txt
+          echo "UV_LOCKED=1" >> $GITHUB_ENV
+          echo "UV_NO_SYNC=1" >> $GITHUB_ENV
+      - name: Upload dependencies artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dependencies-spider-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          path: requirements-freeze.txt
+          compression-level: 0
+
+      - name: Launch LiteLLM Proxy
+        run: |
+          ./scripts/litellm_run.sh
+        env:
+          AZURE_API_BASE: ${{ secrets.AZURE_GROUP_SUBSCRIPTION_API_BASE }}
+          AZURE_API_KEY: ${{ secrets.AZURE_GROUP_SUBSCRIPTION_API_KEY }}
+
+      - name: Prepare RAG dataset
+        run: |
+          set -euo pipefail
+          cd examples/rag
+          mkdir -p data
+          uv run gdown --fuzzy "https://drive.google.com/file/d/1Pq4Ag8zVoN8gUtLu0LcBfY35Dm5zL0hq/view?usp=drive_link" -O data/dataset_tiny.parquet
+          uv run gdown --fuzzy "https://drive.google.com/file/d/1REXCpRLbeZu1KfWWKhIGEQe_WNHUOBkS/view?usp=drive_link" -O data/chunks_candidate_tiny.pkl
+          uv run gdown --fuzzy "https://drive.google.com/file/d/1f6P-h_8KSRhe5pqDHWbRQWvUhTygfZ-c/view?usp=drive_link" -O data/index_hnsw_faiss_n32e40_tiny.index
+
+      - name: Run WIKI Retriever MCP Server
+        run: |
+          set -euo pipefail
+          cd examples/rag
+          uv run python wiki_retriever_mcp.py &
+          for i in {1..20}; do
+            sleep 5
+            if nc -z localhost 8099; then
+              echo "MCP server is up!"
+              exit 0
+            else
+              echo "Waiting for MCP server to start..."
+            fi
+          done
+          echo "MCP server failed to start within expected time."
+          exit 1
+
+      - name: Run vLLM Server
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          vllm serve Qwen/Qwen2.5-1.5B-Instruct \
+            --enable-auto-tool-choice \
+            --tool-call-parser hermes \
+            --port 8000 &
+
+          VLLM_READY=0
+          for i in {1..100}; do
+            if curl -sSf http://localhost:8000/v1/models > /dev/null 2>&1; then
+              echo "vLLM server is ready!"
+              VLLM_READY=1
+              break
+            fi
+            echo "Waiting for vLLM server to be ready... (${i})"
+            sleep 5
+          done
+          if [[ "$VLLM_READY" != "1" ]]; then
+            echo "vLLM server failed to start!"
+            exit 1
+          fi
+
+      - name: Run RAG Sanity check
+        run: |
+          set -ex
+          source .venv/bin/activate
+          cd examples/rag
+          uv run python rag_agent.py
+        shell: bash
+
+      - name: Stop vLLM Server
+        run: |
+          set -euo pipefail
+          pkill -f vllm
+          for i in {1..60}; do
+            if ! pgrep -f vllm; then
+              break
+            fi
+            sleep 5
+          done
+
+      - name: RAG training
+        run: |
+          set -ex
+          source .venv/bin/activate
+          cd examples/rag
+          ../../scripts/restart_ray.sh
+          sleep 5
+          PYTHONUNBUFFERED=1 python train_rag.py fast
+          sleep 10
+        shell: bash
+        env:
+          WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
+          WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}
+        id: rag_train
+
+      - name: Validate RAG training
+        run: |
+          set -ex
+          # Allow up to 5 rollouts to fail to produce rewards
+          uv run scripts/validate_example_wandb.py ${{ steps.rag_train.outputs.project_name }} ${{ steps.rag_train.outputs.run_name }} --reward-tolerance 5
+        env:
+          WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
+          WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}

.github/workflows/examples-spider.yml

@@ -14,7 +14,7 @@ on:
 run-name: >-
   ${{ github.event_name == 'repository_dispatch'
       && format(
-        'PR #{0} - Label {1} - {2}',
+        'Spider - PR #{0} - {1} - {2}',
         github.event.client_payload.pull_number,
         github.event.client_payload.ci_label,
         github.event.client_payload.correlation_id
@@ -33,8 +33,7 @@ jobs:
     strategy:
       matrix:
         include:
-        - python-version: '3.10'
-          setup-script: 'legacy'
+        # legacy is omitted because langchain doesn't work with legacy vllm versions
         - python-version: '3.12'
           setup-script: 'stable'
         - python-version: '3.13'
@@ -58,13 +57,13 @@ jobs:
       - name: Sync dependencies (latest)
         run: |
           uv sync --frozen --no-default-groups --extra verl \
-            --group dev --group experiment --group agents --group torch-gpu-stable
+            --group dev --group experiment --group agents --group langchain --group torch-gpu-stable
         if: matrix.setup-script == 'latest'
-      - name: Sync dependencies (stable & legacy)
+      - name: Sync dependencies (stable)
         run: |
           uv sync --frozen --no-default-groups --extra verl \
-            --group dev --group experiment --group agents --group torch-gpu-${{ matrix.setup-script }}
-        if: matrix.setup-script != 'latest'
+            --group dev --group experiment --group agents --group langchain --group torch-gpu-${{ matrix.setup-script }}
+        if: matrix.setup-script == 'stable'
       - name: Freeze dependencies
         run: |
           set -ex
@@ -121,7 +120,7 @@ jobs:
       - name: Validate Spider training
         run: |
           set -ex
-          uv run scripts/validate_example_wandb.py ${{ steps.spider_train.outputs.project_name }} ${{ steps.spider_train.outputs.run_name }}
+          uv run scripts/validate_example_wandb.py ${{ steps.spider_train.outputs.project_name }} ${{ steps.spider_train.outputs.run_name }} --reward-tolerance 5
         env:
           WANDB_BASE_URL: ${{ secrets.MSR_WANDB_BASE_URL }}
           WANDB_API_KEY: ${{ secrets.MSR_WANDB_API_KEY }}

.github/workflows/examples-tinker.yml

@@ -0,0 +1,160 @@
+name: Examples - Tinker
+permissions:
+  contents: read
+on:
+  schedule:
+    # Every day at 3 AM UTC+8
+    - cron: '0 19 * * *'
+
+  workflow_dispatch:
+
+  repository_dispatch:
+    types: [ci-tinker, ci-all]
+
+run-name: >-
+  ${{ github.event_name == 'repository_dispatch'
+      && format(
+        'Tinker - PR #{0} - {1} - {2}',
+        github.event.client_payload.pull_number,
+        github.event.client_payload.ci_label,
+        github.event.client_payload.correlation_id
+      )
+      || format('Tinker - {0}', github.event_name) }}
+
+jobs:
+  tinker:
+    if: >
+      github.event_name != 'repository_dispatch' ||
+      github.event.action == 'ci-tinker' ||
+      github.event.action == 'ci-all'
+    name: Tinker (Python ${{ matrix.python-version }}, ${{ matrix.setup-script }})
+    runs-on: [self-hosted, 1ES.Pool=agl-runner-cpu]
+    timeout-minutes: 150
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.12'
+          setup-script: 'stable'
+        - python-version: '3.13'
+          setup-script: 'latest'
+      fail-fast: false
+    steps:
+      - name: Check disk space
+        run: df -h
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_ref || (github.event.pull_request.number && format('refs/pull/{0}/merge', github.event.pull_request.number)) || github.ref }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade dependencies (latest)
+        run: uv lock --upgrade
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies
+        run: |
+          uv sync --frozen --no-default-groups \
+            --group dev --group experiment --group agents --group torch-cpu --group core-stable --group tinker
+      - name: Freeze dependencies
+        run: |
+          set -euo pipefail
+          uv pip freeze | tee requirements-freeze.txt
+          echo "UV_LOCKED=1" >> $GITHUB_ENV
+          echo "UV_NO_SYNC=1" >> $GITHUB_ENV
+      - name: Upload dependencies artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dependencies-tinker-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          path: requirements-freeze.txt
+          compression-level: 0
+
+      - name: Tinker LLM sanity check
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/tinker
+          # TODO: Currently only test the client tracer implementation.
+          python -m tests.test_tinker_llm
+        shell: bash
+        env:
+          TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }}
+
+      - name: Tinker Hello
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/tinker
+          python hello.py oneclick --ci
+        shell: bash
+        env:
+          TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }}
+
+      - name: Tinker Q20 Evaluate (GPT-4.1)
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/tinker
+          mkdir -p logs
+          python q20_evaluate.py --ci --model gpt-4.1 --output-file logs/q20_evaluate_gpt-4.1.jsonl
+        shell: bash
+        env:
+          OPENAI_BASE_URL: ${{ secrets.AZURE_OPENAI_ENDPOINT_SWEDEN }}
+          OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_SWEDEN }}
+          CREWAI_DISABLE_TELEMETRY: true
+          TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }}
+
+      - name: Tinker Q20 Evaluate (Qwen3-30B-A3B-Instruct-2507)
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/tinker
+          python q20_evaluate.py --ci --model Qwen/Qwen3-30B-A3B-Instruct-2507 --output-file logs/q20_evaluate_qwen3-30b-a3b.jsonl
+        shell: bash
+        env:
+          OPENAI_BASE_URL: ${{ secrets.AZURE_OPENAI_ENDPOINT_SWEDEN }}
+          OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_SWEDEN }}
+          CREWAI_DISABLE_TELEMETRY: true
+          TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }}
+
+      - name: Tinker Q20 Training Dry Run
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/tinker
+          python q20_train.py dryrun --model qwen4b
+        shell: bash
+        env:
+          OPENAI_BASE_URL: ${{ secrets.AZURE_OPENAI_ENDPOINT_SWEDEN }}
+          OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_SWEDEN }}
+          CREWAI_DISABLE_TELEMETRY: true
+          TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }}
+
+      - name: Tinker Q20 Training
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/tinker
+          agl store --port 4747 &
+          sleep 5
+          python q20_train.py runner --n-runners 4 &
+          sleep 5
+          python q20_train.py algo --model qwen4b --ci
+          sleep 5
+
+          pkill -f agl && echo "SIGTERM sent to agl" || echo "No agl process found"
+          while pgrep -f agl; do
+            echo "Waiting for agl to finish..."
+            sleep 5
+          done
+          pkill -f q20_train.py && echo "SIGTERM sent to q20_train.py" || echo "No q20_train.py process found"
+          while pgrep -f q20_train.py; do
+            echo "Waiting for q20_train.py to finish..."
+            sleep 5
+          done
+          echo "q20_train.py has finished."
+        shell: bash
+        env:
+          OPENAI_BASE_URL: ${{ secrets.AZURE_OPENAI_ENDPOINT_SWEDEN }}
+          OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_SWEDEN }}
+          CREWAI_DISABLE_TELEMETRY: true
+          TINKER_API_KEY: ${{ secrets.TINKER_API_KEY }}

.github/workflows/examples-unsloth.yml

@@ -14,7 +14,7 @@ on:
 run-name: >-
   ${{ github.event_name == 'repository_dispatch'
       && format(
-        'PR #{0} - Label {1} - {2}',
+        'Unsloth - PR #{0} - {1} - {2}',
         github.event.client_payload.pull_number,
         github.event.client_payload.ci_label,
         github.event.client_payload.correlation_id

.github/workflows/tests-full.yml

@@ -14,7 +14,7 @@ on:
 run-name: >-
   ${{ github.event_name == 'repository_dispatch'
       && format(
-        'PR #{0} - Label {1} - {2}',
+        'GPU Test - PR #{0} - {1} - {2}',
         github.event.client_payload.pull_number,
         github.event.client_payload.ci_label,
         github.event.client_payload.correlation_id
@@ -47,6 +47,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_ref || (github.event.pull_request.number && format('refs/pull/{0}/merge', github.event.pull_request.number)) || github.ref }}
+
       - uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
@@ -55,11 +56,15 @@ jobs:
         run: uv lock --upgrade
         if: matrix.setup-script == 'latest'
       - name: Sync dependencies (latest)
-        run: uv sync --frozen --no-default-groups --extra apo --group dev --group agents --group torch-gpu-stable
+        run: uv sync --frozen --no-default-groups --extra apo --extra mongo --group dev --group agents --group langchain --group torch-gpu-stable
         if: matrix.setup-script == 'latest'
-      - name: Sync dependencies (stable & legacy)
-        run: uv sync --frozen --no-default-groups --extra apo --group dev --group agents --group torch-gpu-${{ matrix.setup-script }}
-        if: matrix.setup-script != 'latest'
+      - name: Sync dependencies (stable)
+        run: uv sync --frozen --no-default-groups --extra apo --extra mongo --group dev --group agents --group langchain --group torch-gpu-${{ matrix.setup-script }}
+        if: matrix.setup-script == 'stable'
+      # Don't install langchain for legacy dependency because it has conflicts with torch.
+      - name: Sync dependencies (legacy)
+        run: uv sync --frozen --no-default-groups --extra apo --extra mongo --group dev --group agents --group torch-gpu-legacy
+        if: matrix.setup-script == 'legacy'
       - name: Freeze dependencies
         run: |
           set -ex
@@ -69,7 +74,7 @@ jobs:
       - name: Upload dependencies artifact
         uses: actions/upload-artifact@v4
         with:
-          name: dependencies-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          name: dependencies-tests-full-${{ matrix.python-version }}-${{ matrix.setup-script }}
           path: requirements-freeze.txt
           compression-level: 0
 
@@ -81,6 +86,52 @@ jobs:
       - name: Build dashboard
         run: cd dashboard && npm run build
 
+      - name: Setup Docker environments
+        run: |
+          set -euo pipefail
+
+          cd docker
+
+          # Setup data directories
+          ./setup.sh
+
+          # Start Dockers
+          docker compose -f compose.mongo.yml up -d
+
+          SERVICE_NAME=mongo
+          TIMEOUT=60   # seconds
+          SLEEP=2
+
+          cid="$(docker compose -f compose.mongo.yml ps -q "$SERVICE_NAME")"
+          if [ -z "$cid" ]; then
+            echo "Service $SERVICE_NAME is not running"
+            exit 1
+          fi
+
+          echo "Waiting for $SERVICE_NAME to become healthy..."
+          end=$((SECONDS + TIMEOUT))
+
+          while [ "$SECONDS" -lt "$end" ]; do
+            status="$(docker inspect -f '{{.State.Health.Status}}' "$cid")"
+            echo "Current status: $status"
+
+            if [ "$status" = "healthy" ]; then
+              echo "$SERVICE_NAME is healthy ✅"
+              exit 0
+            elif [ "$status" = "unhealthy" ]; then
+              echo "$SERVICE_NAME is unhealthy ❌"
+              docker logs "$cid" || true
+              exit 1
+            fi
+
+            sleep "$SLEEP"
+          done
+
+          echo "Timed out waiting for $SERVICE_NAME to become healthy after ${TIMEOUT}s"
+          docker logs "$cid" || true
+          exit 1
+        shell: bash
+
       - name: Launch LiteLLM Proxy
         run: |
           ./scripts/litellm_run.sh
@@ -95,3 +146,206 @@ jobs:
           PYTEST_ADDOPTS: "--color=yes"
           OPENAI_BASE_URL: http://localhost:12306/
           OPENAI_API_KEY: dummy
+          AGL_TEST_MONGO_URI: mongodb://localhost:27017/?replicaSet=rs0
+
+
+  minimal-examples:
+    if: >
+      github.event_name != 'repository_dispatch' ||
+      github.event.action == 'ci-gpu' ||
+      github.event.action == 'ci-all'
+    name: Minimal Examples with Python ${{ matrix.python-version }} (${{ matrix.setup-script }})
+
+    runs-on: [self-hosted, 1ES.Pool=agl-runner-gpu]
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        include:
+        - python-version: '3.10'
+          setup-script: 'legacy'
+        - python-version: '3.12'
+          setup-script: 'stable'
+        - python-version: '3.13'
+          setup-script: 'latest'
+      fail-fast: false
+    steps:
+      - name: Check GPU status
+        run: nvidia-smi
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_ref || (github.event.pull_request.number && format('refs/pull/{0}/merge', github.event.pull_request.number)) || github.ref }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade dependencies (latest)
+        run: uv lock --upgrade
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies (latest)
+        run: uv sync --frozen --no-default-groups --extra apo --group dev --group agents --group langchain --group torch-gpu-stable
+        if: matrix.setup-script == 'latest'
+      - name: Sync dependencies (stable)
+        run: uv sync --frozen --no-default-groups --extra apo --extra mongo --group dev --group agents --group langchain --group torch-gpu-${{ matrix.setup-script }}
+        if: matrix.setup-script == 'stable'
+      # Don't install langchain for legacy dependency because it has conflicts with torch.
+      - name: Sync dependencies (legacy)
+        run: uv sync --frozen --no-default-groups --extra apo --extra mongo --group dev --group agents --group torch-gpu-legacy
+        if: matrix.setup-script == 'legacy'
+      - name: Freeze dependencies
+        run: |
+          set -ex
+          uv pip freeze | tee requirements-freeze.txt
+          echo "UV_LOCKED=1" >> $GITHUB_ENV
+          echo "UV_NO_SYNC=1" >> $GITHUB_ENV
+      - name: Upload dependencies artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dependencies-minimal-examples-${{ matrix.python-version }}-${{ matrix.setup-script }}
+          path: requirements-freeze.txt
+          compression-level: 0
+
+      - name: Launch LiteLLM Proxy
+        run: |
+          ./scripts/litellm_run.sh
+        env:
+          AZURE_API_BASE: ${{ secrets.AZURE_GROUP_SUBSCRIPTION_API_BASE }}
+          AZURE_API_KEY: ${{ secrets.AZURE_GROUP_SUBSCRIPTION_API_KEY }}
+
+      - name: Write Traces via Otel Tracer
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+          python write_traces.py otel
+          sleep 5
+
+      - name: Write Traces via AgentOps Tracer
+        env:
+          OPENAI_BASE_URL: http://localhost:12306/
+          OPENAI_API_KEY: dummy
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+          python write_traces.py agentops
+          sleep 5
+
+      - name: Write Traces via Otel Tracer with Client
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+          agl store --port 45993 --log-level DEBUG &
+          sleep 5
+          python write_traces.py otel --use-client
+          pkill -f agl && echo "SIGTERM sent to agl" || echo "No agl process found"
+          while pgrep -f agl; do
+            echo "Waiting for agl to finish..."
+            sleep 5
+          done
+
+      - name: Write Traces via AgentOps Tracer with Client
+        env:
+          OPENAI_BASE_URL: http://localhost:12306/
+          OPENAI_API_KEY: dummy
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+          agl store --port 45993 --log-level DEBUG &
+          sleep 5
+          python write_traces.py agentops --use-client
+          pkill -f agl && echo "SIGTERM sent to agl" || echo "No agl process found"
+          while pgrep -f agl; do
+            echo "Waiting for agl to finish..."
+            sleep 5
+          done
+
+      - name: vLLM Server
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+          python vllm_server.py Qwen/Qwen2.5-0.5B-Instruct
+
+      - name: LLM Proxy (OpenAI backend)
+        env:
+          OPENAI_API_BASE: http://localhost:12306/
+          OPENAI_API_KEY: dummy
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+
+          python llm_proxy.py openai gpt-4.1-mini &
+
+          LLM_PROXY_READY=0
+          for attempt in $(seq 1 30); do
+            if curl -sSf http://localhost:43886/health > /dev/null 2>&1; then
+              LLM_PROXY_READY=1
+              break
+            fi
+            sleep 2
+          done
+          if [[ "$LLM_PROXY_READY" != "1" ]]; then
+            echo "LLM proxy failed to become healthy" >&2
+            exit 1
+          fi
+
+          python llm_proxy.py test gpt-4.1-mini
+
+          pkill -f llm_proxy.py && echo "SIGTERM sent to llm_proxy.py" || echo "No llm_proxy.py process found"
+          while pgrep -f llm_proxy.py; do
+            echo "Waiting for llm_proxy.py to finish..."
+            sleep 5
+          done
+
+      - name: LLM Proxy (vLLM backend)
+        if: matrix.setup-script != 'legacy'  # Skip if return_token_ids is not supported
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+          python llm_proxy.py vllm Qwen/Qwen2.5-0.5B-Instruct &
+
+          LLM_PROXY_READY=0
+          for attempt in $(seq 1 30); do
+            if curl -sSf http://localhost:43886/health > /dev/null 2>&1; then
+              LLM_PROXY_READY=1
+              break
+            fi
+            sleep 2
+          done
+          if [[ "$LLM_PROXY_READY" != "1" ]]; then
+            echo "LLM proxy failed to become healthy" >&2
+            exit 1
+          fi
+
+          python llm_proxy.py test Qwen/Qwen2.5-0.5B-Instruct
+
+          pkill -f llm_proxy.py && echo "SIGTERM sent to llm_proxy.py" || echo "No llm_proxy.py process found"
+          while pgrep -f llm_proxy.py; do
+            echo "Waiting for llm_proxy.py to finish..."
+            sleep 5
+          done
+
+      - name: MultiMetrics backend example
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          cd examples/minimal
+          python write_metrics.py --duration 8 --prom-port 9105 --prom-host 0.0.0.0 2>&1 | tee metrics.log &
+          pid=$!
+
+          for attempt in $(seq 1 20); do
+            if curl -sSf http://localhost:9105/metrics | grep -q minimal_requests_total; then
+              echo "Metrics endpoint responding"
+              wait $pid
+              cat metrics.log
+              exit 0
+            fi
+            sleep 1
+          done
+
+          echo "Metrics endpoint did not respond"
+          exit 1

.github/workflows/tests.yml

@@ -37,12 +37,14 @@ jobs:
           uv sync --frozen \
             --extra apo \
             --extra verl \
+            --extra mongo \
             --group dev \
             --group torch-cpu \
             --group torch-stable \
             --group trl \
             --group tinker \
             --group agents \
+            --group langchain \
             --no-default-groups
         if: matrix.setup == 'slow'
       # This pre-commit skips JavaScript on purpose.
@@ -138,10 +140,10 @@ jobs:
         run: uv lock --upgrade
         if: matrix.setup-script == 'latest'
       - name: Sync dependencies (latest)
-        run: uv sync --frozen --no-default-groups --extra apo --group dev --group agents --group core-stable
+        run: uv sync --frozen --no-default-groups --extra apo --group dev --group agents --group langchain --group core-stable
         if: matrix.setup-script == 'latest'
       - name: Sync dependencies (stable & legacy)
-        run: uv sync --frozen --no-default-groups --extra apo --group dev --group agents --group core-${{ matrix.setup-script }}
+        run: uv sync --frozen --no-default-groups --extra apo --group dev --group agents --group langchain --group core-${{ matrix.setup-script }}
         if: matrix.setup-script != 'latest'
       - name: Freeze dependencies
         run: |
@@ -166,7 +168,7 @@ jobs:
 
       - name: Run tests
         run: |
-          uv run pytest -v --durations=0 tests
+          uv run pytest -v --durations=0 tests -m "not mongo"
         env:
           PYTEST_ADDOPTS: "--color=yes"
 

.gitignore

@@ -213,3 +213,6 @@ agentlightning/dashboard/**/*.css
 agentlightning/dashboard/**/*.js
 agentlightning/dashboard/**/*.html
 agentlightning/dashboard/**/*.svg
+
+# Docker data
+docker/data/

AGENTS.md

@@ -0,0 +1,38 @@
+# Repository Guidelines
+
+## Architecture Overview
+Agent Lightning runs through a continuous loop: runners and tracers emit spans, `LightningStore` (`agentlightning/store/`) keeps them synchronized, and algorithms in `agentlightning/algorithm/` consume those traces to improve behavior.
+
+## Project Structure & Module Organization
+- `agentlightning/`: adapters, execution stack, training loop, tracer, reward logic, and the `agl` CLI.
+- `docs/` & `examples/`: narrative and procedural docs (assets in `docs/assets/`, navigation in `mkdocs.yml`) plus runnable workflows whose READMEs point to their companion how-to guides. `docs/how-to` covers task-focused instructions, while `docs/tutorials` explains concepts and subsystems.
+- `dashboard/`, `scripts/`, `tests/`: UI bundles, release/dataset/CI automation, and mirrored coverage of the runtime tree. Record download steps rather than committing binaries.
+
+## Build, Test, and Development Commands
+- `uv sync --group dev` — provision tooling once per environment.
+- `uv run --no-sync pytest -v` — execute the full suite; add a path or `-k expr` to narrow the run.
+- `uv run --no-sync pyright` — enforce static typing parity with CI.
+- `uv run --no-sync pre-commit run --all-files --show-diff-on-failure` and `uv run --no-sync mkdocs build --strict` — keep formatting tidy and documentation valid.
+Always commit the refreshed `uv.lock` when dependencies shift, and mention optional groups (VERL, APO, GPU) in PR notes.
+
+## Coding Style & Naming Conventions
+- Target `requires-python >= 3.10`, four-space indentation, 120-character lines (though docstrings may run longer), and formatter-owned diffs (Black + isort, `black` profile). Use `snake_case` for modules, functions, and variables; `PascalCase` for classes and React components; lowercase hyphenation for CLI flags, branch names, and TypeScript filenames.
+- Maintain exhaustive type hints (pyright enforces them) and prefer shared dataclasses or Pydantic models from `agentlightning.types`.
+- Author Google-style docstrings for new modules or public methods—succinct descriptions, no redundant type info, no redundant `Key features/components` bullet points, and `[][]` syntax for cross-references.
+- Writing logs is encouraged, especially for long functions with multiple steps and try-except blocks that catch all exceptions. Use `logging.getLogger(__name__)` to get loggers. Distinguish between DEBUG, INFO, WARNING, and ERROR logs.
+
+## Testing Guidelines
+- Mirror runtime directories under `tests/` and match filenames for quick traceability.
+- Parametrize pytest cases and apply markers (`openai`, `gpu`, `agentops`, `mongo`, `llmproxy`) so optional suites can be skipped via selectors like `-m "not mongo"` yet still exercised in CI.
+- Lean on fixtures, favor real stores/spans/agents over mocks, and drive coverage across the majority of branches.
+- If an imported module is missing from the environment, check whether `uv sync` has been run with the right groups. Do not make stubs for external dependencies unless necessary.
+
+## Example Contributions
+- Ship each example with a README that includes smoke-test instructions so maintainers can validate quickly. The README must contain an "Included Files" section summarizing every file and its role.
+- Keep runnable example modules self-contained with a module-level docstring describing CLI usage. Document important or educational classes/functions with targeted docstrings and inline comments where clarity matters.
+- Add a CI workflow per example named `examples-<name>.yml` in `.github/workflows/`. Register it in `badge-<name>.yml`, `badge-examples.yml`, and `badge-latest.yml` when applicable so badges stay accurate.
+
+## Commit & Pull Request Guidelines
+- Branch from a fresh `main` using `feature/<slug>`, `fix/<slug>`, `docs/<slug>`, or `chore/<slug>`.
+- Write imperative, scoped commits, reference issues with `Fixes #123`, and rerun pre-commit plus the relevant pytest/doc builds before pushing.
+- Use PR descriptions to summarize intent, list verification commands, call out dependency or docs-navigation updates, and link new docs/examples via `mkdocs.yml` or `examples/README.md`. Include logs for dashboard changes.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

README.md

@@ -4,7 +4,7 @@
 
 # Agent Lightning⚡
 
-[![Test](https://github.com/microsoft/agent-lightning/actions/workflows/tests-full.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/tests-full.yml)
+[![Unit Tests](https://github.com/microsoft/agent-lightning/actions/workflows/badge-unit.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/badge-unit.yml)
 [![Documentation](https://img.shields.io/badge/GitHub%20Pages-Documentation-blue)](https://microsoft.github.io/agent-lightning/)
 [![PyPI version](https://badge.fury.io/py/agentlightning.svg)](https://badge.fury.io/py/agentlightning)
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
@@ -34,6 +34,12 @@ Read more on our [documentation website](https://microsoft.github.io/agent-light
 pip install agentlightning
 ```
 
+For the latest nightly build (cutting-edge features), you can install from Test PyPI:
+
+```bash
+pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ agentlightning
+```
+
 Please refer to our [installation guide](https://microsoft.github.io/agent-lightning/stable/tutorials/installation/) for more details.
 
 To start using Agent-lightning, check out our [documentation](https://microsoft.github.io/agent-lightning/) and [examples](./examples).
@@ -69,10 +75,11 @@ No rewrites, no lock-in, just a clear path from first rollout to steady improvem
 | Workflow | Status |
 |----------|--------|
 | CPU Tests | [![tests workflow status](https://github.com/microsoft/agent-lightning/actions/workflows/tests.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/tests.yml) |
-| GPU Tests | [![tests-full workflow status](https://github.com/microsoft/agent-lightning/actions/workflows/tests-full.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/tests-full.yml) |
+| Full Tests | [![tests summary workflow status](https://github.com/microsoft/agent-lightning/actions/workflows/badge-unit.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/badge-unit.yml) |
+| UI Tests | [![UI Tests](https://github.com/microsoft/agent-lightning/actions/workflows/dashboard.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/dashboard.yml) |
 | Examples Integration | [![examples summary workflow status](https://github.com/microsoft/agent-lightning/actions/workflows/badge-examples.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/badge-examples.yml) |
 | Latest Dependency Compatibility | [![latest summary workflow status](https://github.com/microsoft/agent-lightning/actions/workflows/badge-latest.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/badge-latest.yml) |
-| Legacy Examples Compatibility | [![examples compatibility workflow status](https://github.com/microsoft/agent-lightning/actions/workflows/examples-compat.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/examples-compat.yml) |
+| Legacy Examples Compatibility | [![compat summary workflow status](https://github.com/microsoft/agent-lightning/actions/workflows/badge-compat.yml/badge.svg)](https://github.com/microsoft/agent-lightning/actions/workflows/badge-compat.yml) |
 
 ## ⚡ Citation
 
@@ -92,7 +99,7 @@ If you find Agent Lightning useful in your research or projects, please cite our
 
 ## ⚡ Contributing
 
-This project welcomes contributions and suggestions. Start by reading the [Contributing Guide](docs/community/contributing.md) for environment setup, branching conventions, and pull request expectations. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+This project welcomes contributions and suggestions. Start by reading the [Contributing Guide](docs/community/contributing.md) for recommended contribution points, environment setup, branching conventions, and pull request expectations. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
 
 When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
 

agentlightning/__init__.py

@@ -1,16 +1,19 @@
 # Copyright (c) Microsoft. All rights reserved.
 
-__version__ = "0.2.2"
+__version__ = "0.3.0"
 
 from .adapter import *
 from .algorithm import *
 from .client import AgentLightningClient, DevTaskLoader  # deprecated  # type: ignore
 from .config import *
 from .emitter import *
+from .env_var import *
 from .execution import *
 from .litagent import *
 from .llm_proxy import *
-from .logging import *
+from .logging import configure_logger  # deprecated  # type: ignore
+from .logging import setup as setup_logging  # type: ignore
+from .logging import setup_module as setup_module_logging  # type: ignore
 from .runner import *
 from .server import AgentLightningServer  # deprecated  # type: ignore
 from .store import *

agentlightning/adapter/base.py

@@ -1,6 +1,6 @@
 # Copyright (c) Microsoft. All rights reserved.
 
-from typing import Generic, List, TypeVar
+from typing import Generic, Sequence, TypeVar
 
 from opentelemetry.sdk.trace import ReadableSpan
 
@@ -66,7 +66,7 @@ class Adapter(Generic[T_from, T_to]):
         raise NotImplementedError("Adapter.adapt() is not implemented")
 
 
-class OtelTraceAdapter(Adapter[List[ReadableSpan], T_to], Generic[T_to]):
+class OtelTraceAdapter(Adapter[Sequence[ReadableSpan], T_to], Generic[T_to]):
     """Base class for adapters that convert OpenTelemetry trace spans into other formats.
 
     This specialization of [`Adapter`][agentlightning.Adapter] expects a list of
@@ -84,7 +84,7 @@ class OtelTraceAdapter(Adapter[List[ReadableSpan], T_to], Generic[T_to]):
     """
 
 
-class TraceAdapter(Adapter[List[Span], T_to], Generic[T_to]):
+class TraceAdapter(Adapter[Sequence[Span], T_to], Generic[T_to]):
     """Base class for adapters that convert trace spans into other formats.
 
     This class specializes [`Adapter`][agentlightning.Adapter] for working with

agentlightning/adapter/messages.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import json
 from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Dict, Generator, Iterable, List, Optional, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, Generator, Iterable, List, Optional, Sequence, TypedDict, Union, cast
 
 from pydantic import TypeAdapter
 
@@ -208,7 +208,7 @@ class TraceToMessages(TraceAdapter[List[OpenAIMessages]]):
         children of the associated completion span.
     """
 
-    def get_tool_calls(self, completion: Span, all_spans: List[Span], /) -> Iterable[Dict[str, Any]]:
+    def get_tool_calls(self, completion: Span, all_spans: Sequence[Span], /) -> Iterable[Dict[str, Any]]:
         """Yield tool call payloads for a completion span.
 
         Args:
@@ -231,7 +231,7 @@ class TraceToMessages(TraceAdapter[List[OpenAIMessages]]):
             if tool_call:
                 yield tool_call
 
-    def adapt(self, source: List[Span], /) -> List[OpenAIMessages]:
+    def adapt(self, source: Sequence[Span], /) -> List[OpenAIMessages]:
         """Transform trace spans into OpenAI chat payloads.
 
         Args:

agentlightning/adapter/triplet.py

@@ -6,12 +6,13 @@ import json
 import logging
 import re
 from enum import Enum
-from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast
 
 from opentelemetry.sdk.trace import ReadableSpan
 from pydantic import BaseModel
 
-from agentlightning.types import Span, SpanNames, Triplet
+from agentlightning.emitter.reward import get_reward_value
+from agentlightning.types import Span, Triplet
 
 from .base import TraceAdapter
 
@@ -313,24 +314,11 @@ class TraceTree:
         Returns:
             Dictionary containing reward metadata, or an empty dictionary when no reward is found.
         """
-        for key in [
-            "agentops.task.output",  # newer versions of agentops
-            "agentops.entity.output",
-        ]:
-            output = self.span.attributes.get(key)  # type: ignore
-            if output:
-                if isinstance(output, dict):
-                    return output
-                elif isinstance(output, str):
-                    try:
-                        return json.loads(output)
-                    except json.JSONDecodeError:
-                        return {}
-
-        # Latest emit reward format
-        if self.span.name == SpanNames.REWARD.value and self.span.attributes:
-            return {"type": "reward", "value": self.span.attributes.get("reward", None)}
-        return {}
+        reward_value = get_reward_value(self.span)
+        if reward_value is not None:
+            return {"type": "reward", "value": reward_value}
+        else:
+            return {}
 
     def is_reward_span(self) -> bool:
         """Return whether the span explicitly encodes a reward.
@@ -670,7 +658,7 @@ class TracerTraceToTriplet(TraceToTripletBase):
         trace_tree.visualize(filename, interested_span_match=interested_span_match)
         return trace_tree
 
-    def adapt(self, source: Union[List[Span], List[ReadableSpan]], /) -> List[Triplet]:  # type: ignore
+    def adapt(self, source: Union[Sequence[Span], Sequence[ReadableSpan]], /) -> List[Triplet]:  # type: ignore
         """Convert tracer spans into [`Triplet`][agentlightning.Triplet] trajectories.
 
         Args:
@@ -776,31 +764,14 @@ class LlmProxyTraceToTriplet(TraceToTripletBase):
 
     def _maybe_reward_value(self, span: Span) -> Optional[float]:
         """Parse reward from typical AgentOps payloads or explicit reward spans."""
-        attrs = span.attributes or {}
-
-        # AgentOps new/old keys
-        for k in ("agentops.task.output", "agentops.entity.output"):
-            v = attrs.get(k)
-            v = self._literal_eval_maybe(v)
-            if isinstance(v, dict) and cast(Dict[str, Any], v).get("type") == "reward":
-                rv = cast(Dict[str, Any], v).get("value", None)
-                if rv is None or isinstance(rv, (int, float)):
-                    return None if rv is None else float(rv)
-
-        # Explicit reward span
-        if span.name == SpanNames.REWARD.value:
-            rv = attrs.get("reward", None)
-            if rv is None or isinstance(rv, (int, float)):
-                return None if rv is None else float(rv)
-
-        return None
+        return get_reward_value(span)
 
     def _request_id_from_attrs(self, attrs: Dict[str, Any]) -> Optional[str]:
         # Prefer OpenAI-like id if present, else proxy raw id.
         rid = attrs.get("gen_ai.response.id") or attrs.get("llm.hosted_vllm.id")
         return str(rid) if isinstance(rid, str) and rid else None
 
-    def adapt(self, source: List[Span], /) -> List[Triplet]:  # type: ignore
+    def adapt(self, source: Sequence[Span], /) -> List[Triplet]:  # type: ignore
         """Convert LLM Proxy spans into [`Triplet`][agentlightning.Triplet] trajectories.
 
         Args:

agentlightning/algorithm/fast.py

@@ -143,7 +143,7 @@ class Baseline(FastAlgorithm):
         store = self.get_store()
 
         for index in train_indices + val_indices:
-            queuing_rollouts = await store.query_rollouts(status=["queuing", "requeuing"])
+            queuing_rollouts = await store.query_rollouts(status_in=["queuing", "requeuing"])
             if len(queuing_rollouts) <= 1:
                 # Only enqueue a new rollout when there is at most 1 rollout in the queue.
                 sample = dataset[index]
@@ -222,7 +222,7 @@ class Baseline(FastAlgorithm):
                     f"Processing index {index}. {len(train_indices)} train indices and {len(val_indices)} val indices in total."
                 )
                 while True:
-                    queuing_rollouts = await store.query_rollouts(status=["queuing", "requeuing"])
+                    queuing_rollouts = await store.query_rollouts(status_in=["queuing", "requeuing"])
                     if len(queuing_rollouts) <= self.max_queue_length:
                         # Only enqueue a new rollout when there is at most "max_queue_length" rollout in the queue.
                         sample = concatenated_dataset[index]

agentlightning/cli/store.py

@@ -9,7 +9,7 @@ import asyncio
 import logging
 from typing import Iterable
 
-from agentlightning.logging import configure_logger
+from agentlightning import setup_logging
 from agentlightning.store.client_server import LightningStoreServer
 from agentlightning.store.memory import InMemoryLightningStore
 
@@ -18,6 +18,7 @@ logger = logging.getLogger(__name__)
 
 def main(argv: Iterable[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description="Run a LightningStore server")
+    parser.add_argument("--host", default="0.0.0.0", help="Host to bind the server to")
     parser.add_argument("--port", type=int, default=4747, help="Port to run the server on")
     parser.add_argument(
         "--cors-origin",
@@ -25,16 +26,68 @@ def main(argv: Iterable[str] | None = None) -> int:
         action="append",
         help="Allowed CORS origin. Repeat for multiple origins. Use '*' to allow all origins.",
     )
+    parser.add_argument(
+        "--log-level",
+        default="INFO",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        help="Configure the logging level for the store.",
+    )
+    parser.add_argument(
+        "--prometheus",
+        action="store_true",
+        help="Enable Prometheus metrics.",
+    )
+    parser.add_argument(
+        "--n-workers",
+        default=1,
+        type=int,
+        help=(
+            "Number of workers to run in the server. When it's greater than 1, the server will be run using `mp` launch mode. "
+            "Only applicable for zero-copy stores such as MongoDB backend."
+        ),
+    )
+
+    parser.add_argument(
+        "--backend",
+        choices=["memory", "mongo"],
+        default="memory",
+        help="Backend to use for the store.",
+    )
+    parser.add_argument(
+        "--mongo-uri",
+        default="mongodb://localhost:27017/?replicaSet=rs0",
+        help="MongoDB URI to use for the store. Applicable only if --backend is 'mongo'.",
+    )
+
     args = parser.parse_args(list(argv) if argv is not None else None)
 
-    configure_logger()
+    setup_logging(args.log_level)
 
-    store = InMemoryLightningStore()
+    if args.backend == "memory":
+        store = InMemoryLightningStore(
+            prometheus=args.prometheus, thread_safe=True
+        )  # Using thread_safe store for server
+    elif args.backend == "mongo":
+        from agentlightning.store.mongo import MongoLightningStore
+
+        store = MongoLightningStore(client=args.mongo_uri, prometheus=args.prometheus)
+    else:
+        raise ValueError(f"Invalid backend: {args.backend}")
+
+    if args.n_workers > 1:
+        logger.info(f"Running the server using `mp` launch mode with {args.n_workers} workers.")
+        launch_mode = "mp"
+    else:
+        logger.info("Running the server using `asyncio` launch mode.")
+        launch_mode = "asyncio"
     server = LightningStoreServer(
         store,
-        host="0.0.0.0",
+        host=args.host,
         port=args.port,
         cors_allow_origins=args.cors_origins,
+        launch_mode=launch_mode,
+        prometheus=args.prometheus,
+        n_workers=args.n_workers,
     )
     try:
         asyncio.run(server.run_forever())

agentlightning/emitter/__init__.py

@@ -1,25 +1,32 @@
 # Copyright (c) Microsoft. All rights reserved.
 
+from .annotation import emit_annotation, operation
 from .exception import emit_exception
-from .message import emit_message
-from .object import emit_object
+from .message import emit_message, get_message_value
+from .object import emit_object, get_object_value
 from .reward import (
     emit_reward,
     find_final_reward,
     find_reward_spans,
     get_reward_value,
+    get_rewards_from_span,
     is_reward_span,
     reward,
 )
 
 __all__ = [
     "reward",
+    "operation",
     "emit_reward",
     "get_reward_value",
+    "get_rewards_from_span",
     "is_reward_span",
     "find_reward_spans",
     "find_final_reward",
     "emit_message",
     "emit_object",
     "emit_exception",
+    "emit_annotation",
+    "get_message_value",
+    "get_object_value",
 ]

agentlightning/emitter/annotation.py

@@ -0,0 +1,364 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Helpers for emitting annotation/operation spans."""
+
+import asyncio
+import functools
+import inspect
+import json
+import logging
+from types import TracebackType
+from typing import (
+    Any,
+    Callable,
+    ContextManager,
+    Dict,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+)
+
+from opentelemetry import trace
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.trace import Status, StatusCode
+
+from agentlightning.semconv import AGL_ANNOTATION, AGL_OPERATION, LightningSpanAttributes
+from agentlightning.utils.otel import flatten_attributes, get_tracer
+
+_FnType = TypeVar("_FnType", bound=Callable[..., Any])
+
+logger = logging.getLogger(__name__)
+
+
+def emit_annotation(annotation: Dict[str, Any], propagate: bool = True) -> ReadableSpan:
+    """Emit a new annotation span.
+
+    This is the underlying implementation of [`emit_reward`][agentlightning.emit_reward].
+
+    Annotation spans are used to annotate a specific event or a part of rollout.
+    See [semconv][agentlightning.semconv] for conventional annotation keys in Agent-lightning.
+
+    If annotations contain nested dicts, they will be flattened before emitting.
+    Complex objects will lead to emitting failures.
+
+    Args:
+        annotation: Dictionary containing annotation key-value pairs.
+            Representatives are rewards, tags, and metadata.
+        propagate: Whether to propagate the span to exporters automatically.
+    """
+    annotation_attributes = flatten_attributes(annotation)
+    if any(not isinstance(v, (str, int, float, bool, bytes)) for v in annotation_attributes.values()):
+        raise TypeError("All annotation attributes must be primitive types (str, int, float, bool, bytes)")
+
+    # TODO: this should use a tracer from current context rather than the singleton
+    tracer = get_tracer(use_active_span_processor=propagate)
+    span = tracer.start_span(
+        AGL_ANNOTATION,
+        attributes=annotation_attributes,
+    )
+    logger.debug("Emitting annotation span with keys %s", annotation_attributes)
+    with span:
+        pass
+    if not isinstance(span, ReadableSpan):
+        raise ValueError(f"Span is not a ReadableSpan: {span}")
+
+    return span
+
+
+def _safe_json_dump(obj: Any) -> str:
+    """Serialize an object to JSON, falling back to ``str(obj)`` if needed.
+
+    Args:
+        obj: Object to be serialized.
+
+    Returns:
+        The JSON-encoded string representation of the object, or its string
+        representation if JSON encoding fails.
+    """
+    try:
+        return json.dumps(obj, default=str, ensure_ascii=False)
+    except Exception:
+        return str(obj)
+
+
+class OperationContext:
+    """Context manager and decorator for tracing operations.
+
+    This class manages an OpenTelemetry span for a logical unit of work. It can
+    be used either:
+
+    * As a decorator, in which case inputs and outputs are inferred
+      automatically from the wrapped function's signature.
+    * As a context manager, in which case inputs and outputs can be recorded
+      explicitly via :meth:`set_input` and :meth:`set_output`.
+
+    Attributes:
+        name: Human-readable span name.
+        initial_attributes: Attributes applied when the span is created.
+        tracer: OpenTelemetry tracer used to create spans.
+        span: The currently active span, if any.
+    """
+
+    def __init__(self, name: str, attributes: Dict[str, Any], *, propagate: bool = True) -> None:
+        """Initialize a new operation context.
+
+        Args:
+            name: Human-readable name of the span.
+            attributes: Initial attributes attached to the span. Values are
+                JSON-serialized where necessary.
+            propagate: Whether the span should be sent to active exporters.
+        """
+        self.name: str = name
+        self.initial_attributes: Dict[str, Any] = attributes
+        self.propagate: bool = propagate
+        self.tracer: trace.Tracer = get_tracer(use_active_span_processor=propagate)
+        self.span: Optional[trace.Span] = None
+        self._ctx_token: Optional[ContextManager[Any]] = None
+
+    def __enter__(self) -> "OperationContext":
+        """Enter the context manager and start a new span.
+
+        Returns:
+            The current :class:`OperationContext` instance with an active span.
+        """
+        # 1. Start the span with initial attributes (JSON serialized)
+        sanitized_attrs = {
+            k: _safe_json_dump(v) if not isinstance(v, (str, int, float, bool)) else v
+            for k, v in self.initial_attributes.items()
+        }
+
+        self.span = self.tracer.start_span(self.name, attributes=sanitized_attrs)
+        self._ctx_token = trace.use_span(self.span, end_on_exit=True)
+        self._ctx_token.__enter__()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        """Exit the context manager and finish the span.
+
+        Any exception raised inside the context is recorded on the span and the
+        span status is set to error.
+
+        Args:
+            exc_type: Exception type, if an exception occurred.
+            exc_val: Exception instance, if an exception occurred.
+            exc_tb: Traceback object, if an exception occurred.
+        """
+        # 1. Record Exception if present
+        if exc_val and self.span:
+            self.span.record_exception(exc_val)
+            self.span.set_status(Status(StatusCode.ERROR, str(exc_val)))
+
+        # 2. Close span
+        if self._ctx_token:
+            self._ctx_token.__exit__(exc_type, exc_val, exc_tb)
+
+    def set_input(self, *args: Any, **kwargs: Any) -> None:
+        """Record input arguments on the current span.
+
+        Positional arguments are stored under the ``input.args`` attribute,
+        and keyword arguments are stored under ``input.<name>`` attributes.
+
+        This is intended for use inside a ``with operation(...) as op`` block.
+
+        Args:
+            *args: Positional arguments to record.
+            **kwargs: Keyword arguments to record.
+        """
+        if not self.span:
+            return
+
+        if args:
+            self.span.set_attribute("input.args", _safe_json_dump(args))
+        if kwargs:
+            for k, v in kwargs.items():
+                self.span.set_attribute(f"input.{k}", _safe_json_dump(v))
+
+    def set_output(self, output: Any) -> None:
+        """Record the output value on the current span.
+
+        This is intended for use inside a ``with operation(...) as op`` block.
+
+        Args:
+            output: The output value to record.
+        """
+        if not self.span:
+            return
+        self.span.set_attribute("output", _safe_json_dump(output))
+
+    def __call__(self, fn: _FnType) -> _FnType:
+        """Wrap a callable so its execution is traced in a span.
+
+        When used as a decorator, a new span is created for each call to
+        the wrapped function. The bound arguments are recorded as input
+        attributes, the return value is recorded as an output attribute,
+        and any exception is recorded and marks the span as an error.
+
+        Args:
+            fn: The function or coroutine function to wrap.
+
+        Returns:
+            The wrapped callable.
+        """
+        function_name = fn.__name__
+
+        sig = inspect.signature(fn)
+
+        def _record_auto_inputs(span: trace.Span, args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> None:
+            """Bind arguments to signature and log them on the span.
+
+            Args:
+                span: Span on which to record attributes.
+                args: Positional arguments passed to the wrapped callable.
+                kwargs: Keyword arguments passed to the wrapped callable.
+            """
+            try:
+                bound = sig.bind(*args, **kwargs)
+                bound.apply_defaults()
+                for k, v in bound.arguments.items():
+                    span.set_attribute(
+                        f"{LightningSpanAttributes.OPERATION_INPUT.value}.{k}",
+                        _safe_json_dump(v),
+                    )
+            except Exception:
+                span.set_attribute(
+                    f"{LightningSpanAttributes.OPERATION_INPUT.value}.args",
+                    _safe_json_dump(args),
+                )
+                span.set_attribute(
+                    f"{LightningSpanAttributes.OPERATION_INPUT.value}.kwargs",
+                    _safe_json_dump(kwargs),
+                )
+
+        if asyncio.iscoroutinefunction(fn) or inspect.iscoroutinefunction(fn):
+
+            @functools.wraps(fn)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                """Async wrapper that traces the wrapped coroutine."""
+                # Reuse __enter__ logic via 'with self' would share state incorrectly
+                # across concurrent calls. We must create a new span per call.
+                # So we manually reimplement the span logic for the wrapper here.
+
+                sanitized_attrs = {
+                    k: _safe_json_dump(v) if not isinstance(v, (str, int, float, bool)) else v
+                    for k, v in self.initial_attributes.items()
+                }
+
+                with self.tracer.start_as_current_span(self.name, attributes=sanitized_attrs) as span:
+                    span.set_attribute(LightningSpanAttributes.OPERATION_NAME.value, function_name)
+                    _record_auto_inputs(span, args, kwargs)
+                    try:
+                        result = await fn(*args, **kwargs)
+                        span.set_attribute(
+                            LightningSpanAttributes.OPERATION_OUTPUT.value,
+                            _safe_json_dump(result),
+                        )
+                        return result
+                    except Exception as e:
+                        span.record_exception(e)
+                        span.set_status(Status(StatusCode.ERROR, str(e)))
+                        raise
+
+            return cast(_FnType, async_wrapper)
+
+        else:
+
+            @functools.wraps(fn)
+            def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+                """Sync wrapper that traces the wrapped callable."""
+                sanitized_attrs = {
+                    k: _safe_json_dump(v) if not isinstance(v, (str, int, float, bool)) else v
+                    for k, v in self.initial_attributes.items()
+                }
+
+                with self.tracer.start_as_current_span(self.name, attributes=sanitized_attrs) as span:
+                    span.set_attribute(LightningSpanAttributes.OPERATION_NAME.value, function_name)
+                    _record_auto_inputs(span, args, kwargs)
+                    try:
+                        result = fn(*args, **kwargs)
+                        span.set_attribute(
+                            LightningSpanAttributes.OPERATION_OUTPUT.value,
+                            _safe_json_dump(result),
+                        )
+                        return result
+                    except Exception as e:
+                        span.record_exception(e)
+                        span.set_status(Status(StatusCode.ERROR, str(e)))
+                        raise
+
+            return cast(_FnType, sync_wrapper)
+
+
+@overload
+def operation(fn: _FnType, *, propagate: bool = True, **additional_attributes: Any) -> _FnType: ...
+
+
+@overload
+def operation(*, propagate: bool = True, **additional_attributes: Any) -> OperationContext: ...
+
+
+def operation(
+    fn: Optional[_FnType] = None,
+    *,
+    propagate: bool = True,
+    **additional_attributes: Any,
+) -> Union[_FnType, OperationContext]:
+    """Entry point for tracking operations.
+
+    This helper can be used either as a decorator or as a context manager.
+    The span name is fixed to [`AGL_OPERATION`][agentlightning.semconv.AGL_OPERATION];
+    custom span names are not supported. Any keyword arguments are recorded as span attributes.
+
+    Usage as a decorator:
+
+    ```python
+    @operation
+    def func(...):
+        ...
+
+    @operation(category="compute")
+    def func(...):
+        ...
+    ```
+
+    Usage as a context manager:
+
+    ```python
+    with operation(user_id=123) as op:
+        op.set_input(data=data)
+        # ... do work ...
+        op.set_output(result)
+    ```
+
+    Args:
+        fn: When used as `@operation`, this is the wrapped function.
+            When used as `operation(**attrs)`, this should be omitted (or
+            left as `None`) and only keyword attributes are provided.
+        propagate: Whether spans should use the active span processor. When False,
+            spans will stay local and not be exported.
+        **additional_attributes: Additional span attributes to attach at
+            creation time.
+
+    Returns:
+        Either a wrapped callable (when used as a decorator) or an
+        [`OperationContext`][agentlightning.emitter.annotation.OperationContext]
+        (when used as a context manager factory).
+    """
+    # Case 1: Used as @operation (bare decorator or with attributes)
+    if callable(fn):
+        # Create context with fixed name, then immediately wrap the function
+        return OperationContext(AGL_OPERATION, additional_attributes, propagate=propagate)(fn)
+
+    # Case 2: Used as operation(...) / with operation(...)
+    # Custom span names are intentionally not supported; use AGL_OPERATION.
+    if fn is not None:
+        raise ValueError("Custom span names are intentionally not supported when used as a context manager.")
+    return OperationContext(AGL_OPERATION, additional_attributes, propagate=propagate)

agentlightning/emitter/exception.py

@@ -2,43 +2,53 @@
 
 import logging
 import traceback
+from typing import Any, Dict, Optional
 
 from opentelemetry.semconv.attributes import exception_attributes
 
-from agentlightning.types import SpanNames
-
-from .utils import get_tracer
+from agentlightning.semconv import AGL_EXCEPTION
+from agentlightning.utils.otel import get_tracer
 
 logger = logging.getLogger(__name__)
 
 
-def emit_exception(exception: BaseException) -> None:
+def emit_exception(
+    exception: BaseException, attributes: Optional[Dict[str, Any]] = None, propagate: bool = True
+) -> None:
     """Record an exception with OpenTelemetry metadata.
 
+    Classic OpenTelemetry records exceptions in a dedicated logging service.
+    We simplify the model and use trace spans to record exceptions as well.
+
     Args:
         exception: Raised exception instance to serialize into telemetry attributes.
+        attributes: Additional attributes to attach to the exception span.
+        propagate: Whether to propagate the span to exporters automatically.
 
     !!! note
-        The helper validates its input. Non-exception values are ignored to prevent
-        noisy telemetry and indicate programming mistakes via the logger.
+
+        The helper validates its input. If a non-exception value is provided,
+        a TypeError is raised to indicate a programming mistake.
     """
     if not isinstance(exception, BaseException):  # type: ignore
-        logger.error(f"Expected an BaseException instance, got: {type(exception)}. Skip emit_exception.")
-        return
+        raise TypeError(f"Expected a BaseException instance, got: {type(exception)}.")
 
-    tracer = get_tracer()
+    tracer = get_tracer(use_active_span_processor=propagate)
     stacktrace = "".join(traceback.format_exception(type(exception), exception, exception.__traceback__))
-    attributes = {
+    span_attributes = {
         exception_attributes.EXCEPTION_TYPE: type(exception).__name__,
         exception_attributes.EXCEPTION_MESSAGE: str(exception),
         exception_attributes.EXCEPTION_ESCAPED: True,
     }
     if stacktrace.strip():
-        attributes[exception_attributes.EXCEPTION_STACKTRACE] = stacktrace
+        span_attributes[exception_attributes.EXCEPTION_STACKTRACE] = stacktrace
+
+    if attributes:
+        span_attributes.update(attributes)
 
     span = tracer.start_span(
-        SpanNames.EXCEPTION.value,
-        attributes=attributes,
+        AGL_EXCEPTION,
+        attributes=span_attributes,
     )
     logger.debug("Emitting exception span for %s", type(exception).__name__)
     with span:

agentlightning/emitter/message.py

@@ -1,33 +1,55 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 import logging
+from typing import Any, Dict, Optional
 
-from agentlightning.types import SpanAttributeNames, SpanNames
-
-from .utils import get_tracer
+from agentlightning.semconv import AGL_MESSAGE, LightningSpanAttributes
+from agentlightning.types import SpanLike
+from agentlightning.utils.otel import get_tracer
 
 logger = logging.getLogger(__name__)
 
 
-def emit_message(message: str) -> None:
+def emit_message(message: str, attributes: Optional[Dict[str, Any]] = None, propagate: bool = True) -> None:
     """Emit a textual message as an OpenTelemetry span.
 
+    Commonly used for sending debugging and logging messages.
+
     Args:
         message: Human readable message to attach as a span attribute.
+        attributes: Additional attributes to attach to the message span.
+        propagate: Whether to propagate the span to exporters automatically.
 
     !!! note
         OpenTelemetry distinguishes between logs and spans. Emitting the message as a
         span keeps all Agent Lightning telemetry in a single data store for analysis.
     """
     if not isinstance(message, str):  # type: ignore
-        logger.error(f"Message must be a string, got: {type(message)}. Skip emit_message.")
-        return
+        raise TypeError(f"Message must be a string or list of strings, got: {type(message)}.")
 
-    tracer = get_tracer()
+    tracer = get_tracer(use_active_span_processor=propagate)
+    span_attributes = {LightningSpanAttributes.MESSAGE_BODY.value: message}
+    if attributes:
+        span_attributes.update(attributes)
     span = tracer.start_span(
-        SpanNames.MESSAGE.value,
-        attributes={SpanAttributeNames.MESSAGE.value: message},
+        AGL_MESSAGE,
+        attributes=span_attributes,
     )
     logger.debug("Emitting message span with message: %s", message)
     with span:
         pass
+
+
+def get_message_value(span: SpanLike) -> Optional[str]:
+    """Extract the message string from a message span.
+
+    Args:
+        span: Span-like object to extract the message from.
+    """
+    span_attributes = span.attributes or {}
+    if LightningSpanAttributes.MESSAGE_BODY.value not in span_attributes:
+        return None
+    message = span_attributes[LightningSpanAttributes.MESSAGE_BODY.value]
+    if isinstance(message, str):
+        return message
+    raise TypeError(f"Message must be a string, got: {type(message)}.")

agentlightning/emitter/object.py

@@ -1,37 +1,106 @@
 # Copyright (c) Microsoft. All rights reserved.
 
+import base64
 import json
 import logging
-from typing import Any
+from typing import Any, Dict, Optional
 
-from agentlightning.types import SpanAttributeNames, SpanNames
-
-from .utils import get_tracer
+from agentlightning.semconv import AGL_OBJECT, LightningSpanAttributes
+from agentlightning.types import SpanLike
+from agentlightning.utils.otel import full_qualified_name, get_tracer
 
 logger = logging.getLogger(__name__)
 
 
-def emit_object(object: Any) -> None:
+def emit_object(object: Any, attributes: Optional[Dict[str, Any]] = None, propagate: bool = True) -> None:
     """Emit an object's serialized representation as an OpenTelemetry span.
 
     Args:
         object: Data structure to encode as JSON and attach to the span payload.
+        attributes: Additional attributes to attach to the object span.
+        propagate: Whether to propagate the span to exporters automatically.
 
     !!! note
-        The payload must be JSON serializable. Non-serializable objects are ignored and
-        an error is logged to aid debugging.
+        The payload must be JSON serializable. Non-serializable objects will lead to a RuntimeError.
     """
-    try:
-        serialized = json.dumps(object)
-    except (TypeError, ValueError):
-        logger.error(f"Object must be JSON serializable, got: {type(object)}. Skip emit_object.")
-        return
-
-    tracer = get_tracer()
+    span_attributes = encode_object(object)
+    if attributes:
+        span_attributes.update(attributes)
+    tracer = get_tracer(use_active_span_processor=propagate)
     span = tracer.start_span(
-        SpanNames.OBJECT.value,
-        attributes={SpanAttributeNames.OBJECT.value: serialized},
+        AGL_OBJECT,
+        attributes=span_attributes,
     )
-    logger.debug("Emitting object span with payload size %d characters", len(serialized))
+    attr_length = 0
+    if LightningSpanAttributes.OBJECT_JSON.value in span_attributes:
+        attr_length = len(span_attributes[LightningSpanAttributes.OBJECT_JSON.value])
+    elif LightningSpanAttributes.OBJECT_LITERAL.value in span_attributes:
+        attr_length = len(span_attributes[LightningSpanAttributes.OBJECT_LITERAL.value])
+    logger.debug("Emitting object span with payload size %d characters", attr_length)
     with span:
         pass
+
+
+def encode_object(object: Any) -> Dict[str, Any]:
+    """Encode an object as span attributes.
+
+    Args:
+        object: Data structure to encode as JSON.
+    """
+    span_attributes = {}
+    if isinstance(object, (str, int, float, bool)):
+        span_attributes = {
+            LightningSpanAttributes.OBJECT_TYPE.value: type(object).__name__,
+            LightningSpanAttributes.OBJECT_LITERAL.value: str(object),
+        }
+    elif isinstance(object, bytes):
+        b64_encoded = base64.b64encode(object).decode("utf-8")
+        span_attributes = {
+            LightningSpanAttributes.OBJECT_TYPE.value: "bytes",
+            LightningSpanAttributes.OBJECT_LITERAL.value: b64_encoded,
+        }
+    else:
+        try:
+            serialized = json.dumps(object)
+        except (TypeError, ValueError) as exc:
+            raise RuntimeError(f"Object must be JSON serializable, got: {type(object)}.") from exc
+
+        span_attributes = {
+            LightningSpanAttributes.OBJECT_TYPE.value: full_qualified_name(type(object)),  # type: ignore
+            LightningSpanAttributes.OBJECT_JSON.value: serialized,
+        }
+
+    return span_attributes
+
+
+def get_object_value(span: SpanLike) -> Any:
+    """Extract the object payload from an object span.
+
+    Args:
+        span: Span object produced by Agent Lightning emitters.
+    """
+    attributes = span.attributes or {}
+    if LightningSpanAttributes.OBJECT_JSON.value in attributes:
+        serialized = attributes[LightningSpanAttributes.OBJECT_JSON.value]
+        try:
+            return json.loads(serialized)  # type: ignore
+        except (TypeError, ValueError) as exc:
+            raise RuntimeError("Failed to deserialize object JSON from span.") from exc
+    elif LightningSpanAttributes.OBJECT_LITERAL.value in attributes:
+        literal = attributes[LightningSpanAttributes.OBJECT_LITERAL.value]
+        obj_type = attributes.get(LightningSpanAttributes.OBJECT_TYPE.value, "str")
+        if obj_type == "str":
+            return literal
+        elif obj_type == "int":
+            # Let it raise errors if there are any
+            return int(literal)  # type: ignore
+        elif obj_type == "float":
+            return float(literal)  # type: ignore
+        elif obj_type == "bool":
+            return literal.lower() == "true"  # type: ignore
+        elif obj_type == "bytes":
+            return base64.b64decode(literal.encode("utf-8"))  # type: ignore
+        else:
+            raise RuntimeError(f"Unsupported object type for literal deserialization: {obj_type}")
+    else:
+        return None

agentlightning/emitter/reward.py

@@ -23,10 +23,13 @@ from typing import (
 import agentops
 from agentops.sdk.decorators import operation
 from opentelemetry.sdk.trace import ReadableSpan
+from pydantic import TypeAdapter
 
-from agentlightning.types import SpanLike, SpanNames
+from agentlightning.semconv import AGL_ANNOTATION, LightningSpanAttributes, RewardPydanticModel
+from agentlightning.types import SpanLike
+from agentlightning.utils.otel import filter_and_unflatten_attributes
 
-from .utils import get_tracer
+from .annotation import emit_annotation
 
 logger = logging.getLogger(__name__)
 
@@ -34,18 +37,26 @@ __all__ = [
     "reward",
     "emit_reward",
     "get_reward_value",
+    "get_rewards_from_span",
     "is_reward_span",
     "find_reward_spans",
     "find_final_reward",
 ]
 
 
-class RewardSpanData(TypedDict):
+class RewardDimension(TypedDict):
+    """Type representing a single dimension in a multi-dimensional reward."""
+
+    name: str
+    value: float
+
+
+class _RewardSpanData(TypedDict):
     type: Literal["reward"]
     value: Optional[float]
 
 
-FnType = TypeVar("FnType", bound=Callable[..., Any])
+_FnType = TypeVar("_FnType", bound=Callable[..., Any])
 
 
 def _agentops_initialized() -> bool:
@@ -53,7 +64,7 @@ def _agentops_initialized() -> bool:
     return agentops.get_client().initialized
 
 
-def reward(fn: FnType) -> FnType:
+def reward(fn: _FnType) -> _FnType:
     """Decorate a reward function so its outputs are tracked as spans.
 
     The decorator integrates with AgentOps when it is available and falls back to
@@ -70,7 +81,7 @@ def reward(fn: FnType) -> FnType:
         Wrapped callable that preserves the original signature.
     """
 
-    def wrap_result(result: Optional[float]) -> RewardSpanData:
+    def wrap_result(result: Optional[float]) -> _RewardSpanData:
         """Normalize the reward value into the span payload format."""
         if result is None:
             return {"type": "reward", "value": None}
@@ -94,7 +105,7 @@ def reward(fn: FnType) -> FnType:
             result: Optional[float] = None
 
             @operation
-            async def agentops_reward_operation() -> RewardSpanData:
+            async def agentops_reward_operation() -> _RewardSpanData:
                 # The reward function we are interested in tracing
                 # It takes zero inputs and return a formatted dict
                 nonlocal result
@@ -118,7 +129,7 @@ def reward(fn: FnType) -> FnType:
             result: Optional[float] = None
 
             @operation
-            def agentops_reward_operation() -> RewardSpanData:
+            def agentops_reward_operation() -> _RewardSpanData:
                 nonlocal result
                 result = fn(*args, **kwargs)
                 return wrap_result(result)
@@ -129,12 +140,36 @@ def reward(fn: FnType) -> FnType:
         return wrapper  # type: ignore
 
 
-def emit_reward(reward: float) -> ReadableSpan:
+def emit_reward(
+    reward: float | Dict[str, Any],
+    *,
+    primary_key: str | None = None,
+    attributes: Dict[str, Any] | None = None,
+    propagate: bool = True,
+) -> ReadableSpan:
     """Emit a reward value as an OpenTelemetry span.
 
+    Examples:
+        Emit a single-dimensional reward:
+        >>> emit_reward(1.0)
+
+        Emit multi-dimensional rewards:
+        >>> emit_reward({"task_completion": 1.0, "efficiency": 0.8}, primary_key="task_completion")
+
+        Emit a reward with additional attributes (for example linking to another response span):
+        >>> from agentlightning.utils.otel import make_link_attributes
+        >>> emit_reward(0.5, attributes=make_link_attributes({"gen_ai.response.id": "response-123"}))
+
+        Or adding tags onto the reward span:
+        >>> from agentlightning.utils.otel import make_tag_attributes
+        >>> emit_reward(0.7, attributes=make_tag_attributes(["fast", "reliable"]))
+
     Args:
         reward: Numeric reward to record. Integers and booleans are converted to
             floating point numbers for consistency.
+            Use a dictionary to represent a multi-dimensional reward.
+        attributes: Other optional span attributes.
+        propagate: Whether to propagate the span to exporters automatically.
 
     Returns:
         Readable span capturing the recorded reward.
@@ -144,20 +179,34 @@ def emit_reward(reward: float) -> ReadableSpan:
             resulting span is not a [`ReadableSpan`](https://opentelemetry.io/docs/concepts/signals/traces/) instance.
     """
     logger.debug(f"Emitting reward: {reward}")
-    if isinstance(reward, (int, bool)):
-        reward = float(reward)
-    if not isinstance(reward, float):
-        raise ValueError(f"Reward must be a number, got: {type(reward)}")
+    reward_dimensions: List[RewardDimension] = []
+    if isinstance(reward, dict):
+        reward_dict: Dict[str, float] = {}
+        for k, v in reward.items():
+            if isinstance(v, (int, bool)):
+                reward_dict[k] = float(v)
+            elif isinstance(v, float):
+                reward_dict[k] = v
+            else:
+                raise ValueError(f"Reward value must be a number, got: {type(v)} for key {k}")
+        if primary_key is None:
+            raise ValueError("When emitting a multi-dimensional reward as a dict, primary_key must be provided.")
+        if primary_key not in reward_dict:
+            raise ValueError(f"Primary key '{primary_key}' not found in reward dict keys: {list(reward_dict.keys())}")
+        reward_dimensions.append(RewardDimension(name=primary_key, value=reward_dict[primary_key]))
+        for k, v in reward_dict.items():
+            if k != primary_key:
+                reward_dimensions.append(RewardDimension(name=k, value=v))
+    else:
+        if isinstance(reward, (int, bool)):
+            reward = float(reward)
+        elif not isinstance(reward, float):  # pyright: ignore[reportUnnecessaryIsInstance]
+            raise TypeError(f"Reward must be a number, got: {type(reward)}")
+        reward_dimensions.append(RewardDimension(name="primary", value=reward))
 
-    # TODO: This should use the tracer from current context by tracer
-    tracer = get_tracer()
-    span = tracer.start_span(SpanNames.REWARD.value, attributes={"reward": reward})
-    # Do nothing; it's just a number
-    with span:
-        pass
-    if not isinstance(span, ReadableSpan):
-        raise ValueError(f"Span is not a ReadableSpan: {span}")
-    return span
+    return emit_annotation(
+        {LightningSpanAttributes.REWARD.value: reward_dimensions, **(attributes or {})}, propagate=propagate
+    )
 
 
 def get_reward_value(span: SpanLike) -> Optional[float]:
@@ -167,8 +216,14 @@ def get_reward_value(span: SpanLike) -> Optional[float]:
         span: Span object produced by AgentOps or Agent Lightning emitters.
 
     Returns:
-        The reward encoded in the span or `None` when the span does not represent a reward.
+        The primary reward encoded in the span or `None` when the span does not represent a reward.
     """
+    # v0.3+ emit reward format
+    reward_list = get_rewards_from_span(span)
+    if reward_list:
+        # Reward list is ordered and the first element is the primary reward
+        return reward_list[0].value
+
     for key in [
         "agentops.task.output",  # newer versions of agentops
         "agentops.entity.output",
@@ -191,19 +246,45 @@ def get_reward_value(span: SpanLike) -> Optional[float]:
                 return None
             if not isinstance(reward_value, float):
                 logger.error(f"Reward is not a number, got: {type(reward_value)}. This may cause undefined behaviors.")
+            logger.warning(
+                f"Extracted reward {reward_value} from AgentOps. This format is deprecated, please migrate to using `emit_reward`."
+            )
             return cast(float, reward_value)
 
-    # Latest emit reward format
-    if span.name == SpanNames.REWARD.value and span.attributes:
+    # v0.2 emit reward format
+    if span.name == AGL_ANNOTATION and span.attributes:
         reward_value = span.attributes.get("reward", None)
         if reward_value is None:
             return None
         if not isinstance(reward_value, float):
             logger.error(f"Reward is not a number, got: {type(reward_value)}. This may cause undefined behaviors.")
+        logger.warning(
+            f"Extracted reward {reward_value} from a legacy version of reward span. You might have inconsistent agent-lightning versions."
+        )
         return cast(float, reward_value)
+
     return None
 
 
+def get_rewards_from_span(span: SpanLike) -> List[RewardPydanticModel]:
+    """Extract the reward as a list from a span, if available.
+
+    Args:
+        span: Span object produced by AgentOps or Agent Lightning emitters.
+
+    Returns:
+        A list of reward dimensions encoded in the span or an empty list when the span does not represent a reward.
+    """
+    if span.attributes and any(key.startswith(LightningSpanAttributes.REWARD.value) for key in span.attributes):
+        reward_attr = filter_and_unflatten_attributes(
+            cast(Any, span.attributes or {}), LightningSpanAttributes.REWARD.value
+        )
+        recovered_rewards = TypeAdapter(List[RewardPydanticModel]).validate_python(reward_attr)
+        return recovered_rewards
+    else:
+        return []
+
+
 def is_reward_span(span: SpanLike) -> bool:
     """Return ``True`` when the provided span encodes a reward value."""
     maybe_reward = get_reward_value(span)

agentlightning/emitter/utils.py

@@ -1,22 +0,0 @@
-# Copyright (c) Microsoft. All rights reserved.
-
-"""Utilities shared across emitter implementations."""
-
-import opentelemetry.trace as trace_api
-from opentelemetry.trace import get_tracer_provider
-
-
-def get_tracer() -> trace_api.Tracer:
-    """Resolve the OpenTelemetry tracer configured for Agent Lightning.
-
-    Returns:
-        OpenTelemetry tracer tagged with the `agentlightning` instrumentation name.
-
-    Raises:
-        RuntimeError: If OpenTelemetry was not initialized before calling this helper.
-    """
-    if hasattr(trace_api, "_TRACER_PROVIDER") and trace_api._TRACER_PROVIDER is None:  # type: ignore[attr-defined]
-        raise RuntimeError("Tracer is not initialized. Cannot emit a meaningful span.")
-
-    tracer_provider = get_tracer_provider()
-    return tracer_provider.get_tracer("agentlightning")

agentlightning/env_var.py

@@ -0,0 +1,156 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Environment variable managements."""
+
+from __future__ import annotations
+
+import os
+from enum import Enum
+from typing import overload
+
+__all__ = [
+    "LightningEnvVar",
+    "resolve_bool_env_var",
+    "resolve_int_env_var",
+    "resolve_str_env_var",
+]
+
+
+class LightningEnvVar(Enum):
+    """Environment variables for Agent Lightning."""
+
+    AGL_EMITTER_DEBUG = "AGL_EMITTER_DEBUG"
+    """Enable debug logging for the emitter."""
+
+    AGL_MANAGED_STORE = "AGL_MANAGED_STORE"
+    """If yes, the [`ExecutionStrategy`][agentlightning.ExecutionStrategy]
+    constructs LightningStore wrappers automatically. When `False` the provided
+    `store` is passed directly to the bundles, allowing callers to manage
+    store wrappers manually."""
+
+    AGL_CURRENT_ROLE = "AGL_CURRENT_ROLE"
+    """Which side(s) to run in this process. Used in
+    [`ClientServerExecutionStrategy`][agentlightning.ClientServerExecutionStrategy]."""
+
+    AGL_SERVER_HOST = "AGL_SERVER_HOST"
+    """Interface the [`LightningStoreServer`][agentlightning.LightningStoreServer]
+    binds to when running the algorithm bundle locally."""
+
+    AGL_SERVER_PORT = "AGL_SERVER_PORT"
+    """Port the [`LightningStoreServer`][agentlightning.LightningStoreServer] listens to."""
+
+
+_TRUTHY_VALUES = {"1", "true", "yes", "on"}
+_FALSY_VALUES = {"0", "false", "no", "off"}
+
+
+@overload
+def resolve_bool_env_var(env_var: LightningEnvVar, override: bool, fallback: bool) -> bool: ...
+
+
+@overload
+def resolve_bool_env_var(env_var: LightningEnvVar, *, fallback: bool) -> bool: ...
+
+
+@overload
+def resolve_bool_env_var(
+    env_var: LightningEnvVar, override: bool | None = None, fallback: bool | None = None
+) -> bool | None: ...
+
+
+def resolve_bool_env_var(
+    env_var: LightningEnvVar, override: bool | None = None, fallback: bool | None = None
+) -> bool | None:
+    """Resolve a boolean environment variable.
+
+    Args:
+        env_var: The environment variable to resolve.
+        override: Optional override supplied by the caller.
+        fallback: Default value if the environment variable is not set.
+    """
+
+    if override is not None:
+        return override
+
+    env_value = os.getenv(env_var.value)
+    if env_value is None:
+        return fallback
+
+    normalized = env_value.strip().lower()
+    if normalized in _TRUTHY_VALUES:
+        return True
+    if normalized in _FALSY_VALUES:
+        return False
+
+    raise ValueError(f"{env_var.value} must be one of {_TRUTHY_VALUES} or {_FALSY_VALUES}")
+
+
+@overload
+def resolve_int_env_var(env_var: LightningEnvVar, override: int, fallback: int) -> int: ...
+
+
+@overload
+def resolve_int_env_var(env_var: LightningEnvVar, *, fallback: int) -> int: ...
+
+
+@overload
+def resolve_int_env_var(
+    env_var: LightningEnvVar, override: int | None = None, fallback: int | None = None
+) -> int | None: ...
+
+
+def resolve_int_env_var(
+    env_var: LightningEnvVar, override: int | None = None, fallback: int | None = None
+) -> int | None:
+    """Resolve an integer environment variable.
+
+    Args:
+        env_var: The environment variable to resolve.
+        override: Optional override supplied by the caller.
+        fallback: Default value if the environment variable is not set.
+    """
+    if override is not None:
+        return override
+
+    env_value = os.getenv(env_var.value)
+    if env_value is None:
+        return fallback
+
+    try:
+        return int(env_value)
+    except ValueError:
+        raise ValueError(f"{env_var.value} must be an integer")
+
+
+@overload
+def resolve_str_env_var(env_var: LightningEnvVar, override: str, fallback: str) -> str: ...
+
+
+@overload
+def resolve_str_env_var(env_var: LightningEnvVar, *, fallback: str) -> str: ...
+
+
+@overload
+def resolve_str_env_var(
+    env_var: LightningEnvVar, override: str | None = None, fallback: str | None = None
+) -> str | None: ...
+
+
+def resolve_str_env_var(
+    env_var: LightningEnvVar, override: str | None = None, fallback: str | None = None
+) -> str | None:
+    """Resolve a string environment variable.
+
+    Args:
+        env_var: The environment variable to resolve.
+        override: Optional override supplied by the caller.
+        fallback: Default value if the environment variable is not set.
+    """
+    if override is not None:
+        return override
+
+    env_value = os.getenv(env_var.value)
+    if env_value is None:
+        return fallback
+
+    return env_value

agentlightning/execution/base.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import logging
-import os
 from typing import Protocol
 
 from agentlightning.store.base import LightningStore
@@ -13,47 +12,6 @@ from .events import ExecutionEvent
 logger = logging.getLogger(__name__)
 
 
-_TRUTHY_VALUES = {"1", "true", "yes", "on"}
-_FALSY_VALUES = {"0", "false", "no", "off"}
-
-
-def resolve_managed_store_flag(value: bool | None) -> bool:
-    """Determine whether execution helpers should wrap the provided store.
-
-    The helper first honours an explicit `value`. When `None` it falls back
-    to the `AGL_MANAGED_STORE` environment variable, accepting a variety
-    of truthy and falsy spellings. Missing environment configuration defaults to
-    `True` so that higher-level strategies create the appropriate client or
-    server wrappers automatically.
-
-    Args:
-        value: Optional override supplied by the caller.
-
-    Returns:
-        `True` when a managed store should be created around the provided
-        instance, otherwise `False`.
-
-    Raises:
-        ValueError: If `AGL_MANAGED_STORE` is set to an unsupported
-            value.
-    """
-
-    if value is not None:
-        return value
-
-    env_value = os.getenv("AGL_MANAGED_STORE")
-    if env_value is None:
-        return True
-
-    normalized = env_value.strip().lower()
-    if normalized in _TRUTHY_VALUES:
-        return True
-    if normalized in _FALSY_VALUES:
-        return False
-
-    raise ValueError("AGL_MANAGED_STORE must be one of 1, 0, true, false, yes, no, on, or off")
-
-
 class AlgorithmBundle(Protocol):
     """Callable bundle produced by [`Trainer`][agentlightning.Trainer].
 

agentlightning/execution/client_server.py

@@ -9,10 +9,11 @@ import time
 from multiprocessing.context import BaseContext
 from typing import Callable, Iterable, Literal, cast
 
+from agentlightning.env_var import LightningEnvVar, resolve_bool_env_var, resolve_int_env_var, resolve_str_env_var
 from agentlightning.store.base import LightningStore
 from agentlightning.store.client_server import LightningStoreClient, LightningStoreServer
 
-from .base import AlgorithmBundle, ExecutionStrategy, RunnerBundle, resolve_managed_store_flag
+from .base import AlgorithmBundle, ExecutionStrategy, RunnerBundle
 from .events import ExecutionEvent, MultiprocessingEvent
 
 logger = logging.getLogger(__name__)
@@ -67,10 +68,11 @@ class ClientServerExecutionStrategy(ExecutionStrategy):
         server_host: str | None = None,
         server_port: int | None = None,
         n_runners: int = 1,
-        graceful_timeout: float = 5.0,
-        terminate_timeout: float = 5.0,
+        graceful_timeout: float = 10.0,
+        terminate_timeout: float = 10.0,
         main_process: Literal["algorithm", "runner"] = "algorithm",
         managed_store: bool | None = None,
+        allowed_exit_codes: Iterable[int] = (0, -15),
     ) -> None:
         """Configure the strategy.
 
@@ -94,45 +96,33 @@ class ClientServerExecutionStrategy(ExecutionStrategy):
                 LightningStore client/server wrappers automatically. When
                 `False` the provided `store` is passed directly to the
                 bundles, allowing callers to manage store wrappers manually.
+            allowed_exit_codes: Allowed exit codes for subprocesses.
+                By default, runner can exit gracefully with code 0 or terminated
+                by SIGTERM (-15).
         """
-        if role is None:
-            role_env = os.getenv("AGL_CURRENT_ROLE")
-            if role_env is None:
-                # Use both if not specified via env var or argument
-                role = "both"
-            elif role_env not in ("algorithm", "runner", "both"):
-                raise ValueError("role must be one of 'algorithm', 'runner', or 'both'")
-            else:
-                role = role_env
-
-        if server_host is None:
-            server_host = os.getenv("AGL_SERVER_HOST", "localhost")
-
-        if server_port is None:
-            server_port_env = os.getenv("AGL_SERVER_PORT")
-            if server_port_env is None:
-                server_port = 4747
-            else:
-                try:
-                    server_port = int(server_port_env)
-                except ValueError as exc:
-                    raise ValueError("AGL_SERVER_PORT must be an integer") from exc
-
-        self.role = role
+        resolved_role = resolve_str_env_var(LightningEnvVar.AGL_CURRENT_ROLE, override=role, fallback="both")
+        if resolved_role not in ("algorithm", "runner", "both"):
+            raise ValueError("role must be one of 'algorithm', 'runner', or 'both'")
+        self.role: Literal["algorithm", "runner", "both"] = resolved_role
         self.n_runners = n_runners
-        self.server_host = server_host
-        self.server_port = server_port
+        self.server_host = resolve_str_env_var(
+            LightningEnvVar.AGL_SERVER_HOST, override=server_host, fallback="localhost"
+        )
+        self.server_port = resolve_int_env_var(LightningEnvVar.AGL_SERVER_PORT, override=server_port, fallback=4747)
         self.graceful_timeout = graceful_timeout
         self.terminate_timeout = terminate_timeout
         if main_process not in ("algorithm", "runner"):
             raise ValueError("main_process must be 'algorithm' or 'runner'")
         if main_process == "runner":
-            if role != "both":
+            if self.role != "both":
                 raise ValueError("main_process='runner' is only supported when role='both'")
             if n_runners != 1:
                 raise ValueError("main_process='runner' requires n_runners to be 1")
         self.main_process = main_process
-        self.managed_store = resolve_managed_store_flag(managed_store)
+        self.managed_store = resolve_bool_env_var(
+            LightningEnvVar.AGL_MANAGED_STORE, override=managed_store, fallback=True
+        )
+        self.allowed_exit_codes = tuple(allowed_exit_codes)
 
     async def _execute_algorithm(
         self, algorithm: AlgorithmBundle, store: LightningStore, stop_evt: ExecutionEvent
@@ -338,10 +328,10 @@ class ClientServerExecutionStrategy(ExecutionStrategy):
 
     def _check_process_exitcodes(self, processes: Iterable[multiprocessing.Process]) -> None:
         """Raise an error if any managed process exited with a non-zero status."""
-        failed = [p for p in processes if p.exitcode not in (0, None)]
+        failed = [p for p in processes if p.exitcode not in self.allowed_exit_codes + (None,)]
         if failed:
             formatted = ", ".join(f"{p.name or p.pid} (exitcode={p.exitcode})" for p in failed)
-            raise RuntimeError(f"Subprocesses failed: {formatted}")
+            raise RuntimeError(f"Subprocesses failed with unexpected exit codes: {formatted}")
 
     def execute(self, algorithm: AlgorithmBundle, runner: RunnerBundle, store: LightningStore) -> None:
         logger.info(

agentlightning/execution/shared_memory.py

@@ -7,10 +7,11 @@ from contextlib import suppress
 from queue import SimpleQueue
 from typing import Any, Awaitable, Callable, List, Literal, Optional, Tuple
 
+from agentlightning.env_var import LightningEnvVar, resolve_bool_env_var
 from agentlightning.store.base import LightningStore
 from agentlightning.store.threading import LightningStoreThreaded
 
-from .base import AlgorithmBundle, ExecutionStrategy, RunnerBundle, resolve_managed_store_flag
+from .base import AlgorithmBundle, ExecutionStrategy, RunnerBundle
 from .events import ExecutionEvent, ThreadingEvent
 
 logger = logging.getLogger(__name__)
@@ -62,7 +63,9 @@ class SharedMemoryExecutionStrategy(ExecutionStrategy):
         self.join_timeout = join_timeout
         self.graceful_delay = graceful_delay
         self.poll_interval = poll_interval
-        self.managed_store = resolve_managed_store_flag(managed_store)
+        self.managed_store = resolve_bool_env_var(
+            LightningEnvVar.AGL_MANAGED_STORE, override=managed_store, fallback=True
+        )
 
     async def _run_until_completed_or_canceled(self, coro: Awaitable[Any], stop_evt: ExecutionEvent) -> Any:
         """Run `coro` until it finishes or a cooperative stop is requested.

agentlightning/instrumentation/agentops.py

@@ -13,7 +13,8 @@ from agentops.sdk.exporters import AuthenticatedOTLPExporter
 from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.metrics.export import MetricExportResult
-from opentelemetry.sdk.trace.export import SpanExportResult
+
+from agentlightning.utils.otlp import LightningStoreOTLPExporter
 
 logger = logging.getLogger(__name__)
 
@@ -32,25 +33,27 @@ def enable_agentops_service(enabled: bool = True) -> None:
     """
     Enable or disable communication with the AgentOps service.
 
-    False (default): AgentOps exporters and clients will run in local mode
-    and will not attempt to communicate with the remote AgentOps service.
-    True: all exporters and clients will operate in normal mode and send data
-    to the AgentOps service as expected.
+    By default, AgentOps exporters and clients will run in local mode
+    and will NOT attempt to communicate with the remote AgentOps service.
+
+    Args:
+        enabled: If True, enable all AgentOps exporters and clients.
+            All exporters and clients will operate in normal mode and send data
+            to the [AgentOps service](https://www.agentops.ai).
     """
     global _agentops_service_enabled
     _agentops_service_enabled = enabled
-    logger.info(f"Switch set to {enabled} for exporters and clients.")
+    logger.info(f"AgentOps service enabled is set to {enabled}.")
 
 
 def _patch_exporters():
     import agentops.client.api
     import agentops.sdk.core
-    import opentelemetry.exporter.otlp.proto.http.metric_exporter
-    import opentelemetry.exporter.otlp.proto.http.trace_exporter
 
     agentops.sdk.core.AuthenticatedOTLPExporter = BypassableAuthenticatedOTLPExporter  # type: ignore
-    opentelemetry.exporter.otlp.proto.http.metric_exporter.OTLPMetricExporter = BypassableOTLPMetricExporter
-    opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter = BypassableOTLPSpanExporter
+    agentops.sdk.core.OTLPMetricExporter = BypassableOTLPMetricExporter
+    if hasattr(agentops.sdk.core, "OTLPSpanExporter"):
+        agentops.sdk.core.OTLPSpanExporter = BypassableOTLPSpanExporter  # type: ignore
     agentops.client.api.V3Client = BypassableV3Client
     agentops.client.api.V4Client = BypassableV4Client
 
@@ -58,12 +61,11 @@ def _patch_exporters():
 def _unpatch_exporters():
     import agentops.client.api
     import agentops.sdk.core
-    import opentelemetry.exporter.otlp.proto.http.metric_exporter
-    import opentelemetry.exporter.otlp.proto.http.trace_exporter
 
     agentops.sdk.core.AuthenticatedOTLPExporter = AuthenticatedOTLPExporter  # type: ignore
-    opentelemetry.exporter.otlp.proto.http.metric_exporter.OTLPMetricExporter = OTLPMetricExporter
-    opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter = OTLPSpanExporter
+    agentops.sdk.core.OTLPMetricExporter = OTLPMetricExporter
+    if hasattr(agentops.sdk.core, "OTLPSpanExporter"):
+        agentops.sdk.core.OTLPSpanExporter = OTLPSpanExporter  # type: ignore
     agentops.client.api.V3Client = V3Client
     agentops.client.api.V4Client = V4Client
 
@@ -243,18 +245,15 @@ def uninstrument_agentops():
         pass
 
 
-class BypassableAuthenticatedOTLPExporter(AuthenticatedOTLPExporter):
+class BypassableAuthenticatedOTLPExporter(LightningStoreOTLPExporter, AuthenticatedOTLPExporter):
     """
     AuthenticatedOTLPExporter with switchable service control.
+
     When `_agentops_service_enabled` is False, skip export and return success.
     """
 
-    def export(self, *args: Any, **kwargs: Any) -> SpanExportResult:
-        if _agentops_service_enabled:
-            return super().export(*args, **kwargs)
-        else:
-            logger.debug("SwitchableAuthenticatedOTLPExporter is switched off, skipping export.")
-            return SpanExportResult.SUCCESS
+    def should_bypass(self) -> bool:
+        return not _agentops_service_enabled
 
 
 class BypassableOTLPMetricExporter(OTLPMetricExporter):
@@ -271,18 +270,16 @@ class BypassableOTLPMetricExporter(OTLPMetricExporter):
             return MetricExportResult.SUCCESS
 
 
-class BypassableOTLPSpanExporter(OTLPSpanExporter):
+class BypassableOTLPSpanExporter(LightningStoreOTLPExporter):
     """
     OTLPSpanExporter with switchable service control.
     When `_agentops_service_enabled` is False, skip export and return success.
+
+    This is used instead of BypassableAuthenticatedOTLPExporter on legacy AgentOps versions.
     """
 
-    def export(self, *args: Any, **kwargs: Any) -> SpanExportResult:
-        if _agentops_service_enabled:
-            return super().export(*args, **kwargs)
-        else:
-            logger.debug("SwitchableOTLPSpanExporter is switched off, skipping export.")
-            return SpanExportResult.SUCCESS
+    def should_bypass(self) -> bool:
+        return not _agentops_service_enabled
 
 
 class BypassableV3Client(V3Client):

agentlightning/llm_proxy.py

@@ -4,12 +4,15 @@ from __future__ import annotations
 
 import ast
 import asyncio
+import json
 import logging
 import os
 import re
 import tempfile
 import threading
+import time
 from contextlib import asynccontextmanager
+from datetime import datetime
 from typing import (
     Any,
     AsyncGenerator,
@@ -18,8 +21,11 @@ from typing import (
     Dict,
     Iterable,
     List,
+    Literal,
     Optional,
     Sequence,
+    Tuple,
+    Type,
     TypedDict,
     Union,
     cast,
@@ -29,13 +35,19 @@ import litellm
 import opentelemetry.trace as trace_api
 import yaml
 from fastapi import Request, Response
+from fastapi.responses import StreamingResponse
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
 from litellm.proxy.proxy_server import app, save_worker_config  # pyright: ignore[reportUnknownVariableType]
+from litellm.types.utils import CallTypes
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import ReadableSpan
 from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
 from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.types import Scope
 
+from agentlightning.semconv import LightningResourceAttributes
 from agentlightning.types import LLM, ProxyLLM
 from agentlightning.utils.server_launcher import (
     LaunchMode,
@@ -163,6 +175,24 @@ class AddReturnTokenIds(CustomLogger):
         return {**data, "return_token_ids": True}
 
 
+class AddLogprobs(CustomLogger):
+    """LiteLLM logger hook to request logprobs from vLLM.
+
+    This mutates the outgoing request payload to include `logprobs=1`
+    for backends that support logprobs return (e.g., vLLM).
+    """
+
+    async def async_pre_call_hook(self, *args: Any, **kwargs: Any) -> Optional[Union[Exception, str, Dict[str, Any]]]:
+        """Async pre-call hook to adjust request payload."""
+        try:
+            data = _get_pre_call_data(args, kwargs)
+        except Exception as e:
+            return e
+
+        # Ensure logprobs are requested from the backend when supported.
+        return {**data, "logprobs": 1}
+
+
 class LightningSpanExporter(SpanExporter):
     """Buffered OTEL span exporter with subtree flushing and training-store sink.
 
@@ -183,7 +213,7 @@ class LightningSpanExporter(SpanExporter):
     def __init__(self, _store: Optional[LightningStore] = None):
         self._store: Optional[LightningStore] = _store  # this is only for testing purposes
         self._buffer: List[ReadableSpan] = []
-        self._lock: Optional[threading.RLock] = None
+        self._lock: Optional[threading.Lock] = None
         self._loop_lock_pid: Optional[int] = None
 
         # Single dedicated event loop running in a daemon thread.
@@ -192,6 +222,8 @@ class LightningSpanExporter(SpanExporter):
         self._loop: Optional[asyncio.AbstractEventLoop] = None
         self._loop_thread: Optional[threading.Thread] = None
 
+        self._otlp_exporter = OTLPSpanExporter()
+
     def _ensure_loop(self) -> asyncio.AbstractEventLoop:
         """Lazily initialize the event loop and thread on first use.
 
@@ -205,15 +237,15 @@ class LightningSpanExporter(SpanExporter):
             self._loop_thread.start()
         return self._loop
 
-    def _ensure_lock(self) -> threading.RLock:
+    def _ensure_lock(self) -> threading.Lock:
         """Lazily initialize the lock on first use.
 
         Returns:
-            threading.RLock: The initialized lock.
+            threading.Lock: The initialized lock.
         """
         self._clear_loop_and_lock()
         if self._lock is None:
-            self._lock = threading.RLock()
+            self._lock = threading.Lock()
         return self._lock
 
     def _clear_loop_and_lock(self) -> None:
@@ -275,24 +307,18 @@ class LightningSpanExporter(SpanExporter):
         with self._ensure_lock():
             for span in spans:
                 self._buffer.append(span)
-
-        # Run the async flush on our private loop, synchronously from caller's POV.
-        async def _locked_flush():
-            # Take the lock inside the coroutine to serialize with other flushes.
-            with self._ensure_lock():
-                return await self._maybe_flush()
-
-        try:
-            loop = self._ensure_loop()
-            fut = asyncio.run_coroutine_threadsafe(_locked_flush(), loop)
-            fut.result()  # Bubble up any exceptions from the coroutine.
-        except Exception as e:
-            logger.exception("Export flush failed: %s", e)
-            return SpanExportResult.FAILURE
+            default_endpoint = self._otlp_exporter._endpoint  # pyright: ignore[reportPrivateUsage]
+            try:
+                self._maybe_flush()
+            except Exception as e:
+                logger.exception("Export flush failed: %s", e)
+                return SpanExportResult.FAILURE
+            finally:
+                self._otlp_exporter._endpoint = default_endpoint  # pyright: ignore[reportPrivateUsage]
 
         return SpanExportResult.SUCCESS
 
-    async def _maybe_flush(self):
+    def _maybe_flush(self):
         """Flush ready subtrees from the buffer.
 
         Strategy:
@@ -314,11 +340,20 @@ class LightningSpanExporter(SpanExporter):
             if not subtree_spans:
                 continue
 
+            # Store is initialized lazily here in most cases.
             store = self._store or get_active_llm_proxy().get_store()
             if store is None:
                 logger.warning("Store is not set in LLMProxy. Cannot log spans to store.")
                 continue
 
+            # If the store supports OTLP endpoint, use it.
+            if store.capabilities.get("otlp_traces", False):
+                otlp_traces_endpoint = store.otlp_traces_endpoint()
+                self._otlp_exporter._endpoint = otlp_traces_endpoint  # pyright: ignore[reportPrivateUsage]
+                otlp_enabled = True
+            else:
+                otlp_enabled = False
+
             # Merge all custom headers found in the subtree.
             headers_merged: Dict[str, Any] = {}
 
@@ -352,7 +387,9 @@ class LightningSpanExporter(SpanExporter):
                 headers_merged.update(cast(Dict[str, Any], headers))
 
             if not headers_merged:
-                logger.warning(f"No headers found in {len(subtree_spans)} subtree spans. Cannot log to store.")
+                logger.warning(
+                    f"No headers found in {len(subtree_spans)} subtree spans of root {root_span_id}. Cannot log to store."
+                )
                 continue
 
             # Validate and normalize required header fields.
@@ -372,10 +409,34 @@ class LightningSpanExporter(SpanExporter):
             sequence_id_decimal = int(sequence_id)
 
             # Persist each span in the subtree with the resolved identifiers.
-            for span in subtree_spans:
-                await store.add_otel_span(
-                    rollout_id=rollout_id, attempt_id=attempt_id, sequence_id=sequence_id_decimal, readable_span=span
-                )
+            if otlp_enabled:
+                # If store has OTLP support, directly use OTLP exporter and export in batch
+                for span in subtree_spans:
+                    span._resource = span._resource.merge(  # pyright: ignore[reportPrivateUsage]
+                        Resource.create(
+                            {
+                                LightningResourceAttributes.ROLLOUT_ID.value: rollout_id,
+                                LightningResourceAttributes.ATTEMPT_ID.value: attempt_id,
+                                LightningResourceAttributes.SPAN_SEQUENCE_ID.value: sequence_id_decimal,
+                            }
+                        )
+                    )
+                export_result = self._otlp_exporter.export(subtree_spans)
+                if export_result != SpanExportResult.SUCCESS:
+                    raise RuntimeError(f"Failed to export spans via OTLP exporter. Result: {export_result}")
+
+            else:
+                # The old way: store does not support OTLP endpoint
+                for span in subtree_spans:
+                    loop = self._ensure_loop()
+                    add_otel_span_task = store.add_otel_span(
+                        rollout_id=rollout_id,
+                        attempt_id=attempt_id,
+                        sequence_id=sequence_id_decimal,
+                        readable_span=span,
+                    )
+                    fut = asyncio.run_coroutine_threadsafe(add_otel_span_task, loop)
+                    fut.result()  # Bubble up any exceptions from the coroutine.
 
     def _get_root_span_ids(self) -> Iterable[int]:
         """Yield span_ids for root spans currently in the buffer.
@@ -454,6 +515,23 @@ class LightningOpenTelemetry(OpenTelemetry):
 
         super().__init__(config=config)  # pyright: ignore[reportUnknownMemberType]
 
+    async def async_pre_call_deployment_hook(
+        self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] = None
+    ) -> Optional[Dict[str, Any]]:
+        """The root span is sometimes missing (e.g., when Anthropic endpoint is used).
+        It is created in an auth module in LiteLLM. If it's missing, we create it here.
+        """
+        if "metadata" not in kwargs or "litellm_parent_otel_span" not in kwargs["metadata"]:
+            parent_otel_span = self.create_litellm_proxy_request_started_span(  # type: ignore
+                start_time=datetime.now(),
+                headers=kwargs.get("headers", {}),
+            )
+            updated_metadata = {**kwargs.get("metadata", {}), "litellm_parent_otel_span": parent_otel_span}
+
+            return {**kwargs, "metadata": updated_metadata}
+        else:
+            return kwargs
+
 
 class RolloutAttemptMiddleware(BaseHTTPMiddleware):
     """
@@ -499,6 +577,433 @@ class RolloutAttemptMiddleware(BaseHTTPMiddleware):
         return response
 
 
+class MessageInspectionMiddleware(BaseHTTPMiddleware):
+    """Middleware to inspect the request and response bodies.
+
+    It's for debugging purposes. Add it via "message_inspection" middleware alias.
+    """
+
+    async def dispatch(self, request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response:
+        ti = time.time()
+        logger.info(f"Received request with scope: {request.scope}")
+        logger.info(f"Received request with body: {await request.body()}")
+        response = await call_next(request)
+        elapsed = time.time() - ti
+        logger.info(f"Response to request took {elapsed} seconds")
+        logger.info(f"Received response with status code: {response.status_code}")
+        logger.info(f"Received response with body: {response.body}")
+        return response
+
+
+class StreamConversionMiddleware(BaseHTTPMiddleware):
+    """Middleware to convert streaming responses to non-streaming responses.
+
+    Useful for backend that only supports non-streaming responses.
+
+    LiteLLM's OpenTelemetry is also buggy with streaming responses.
+    The conversion will hopefully bypass the bug.
+    """
+
+    async def dispatch(self, request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response:
+        # Only process POST requests to completion endpoints
+        if request.method != "POST":
+            return await call_next(request)
+
+        # Check if it's a chat completions or messages endpoint
+        endpoint_format: Literal["openai", "anthropic", "unknown"] = "unknown"
+        if request.url.path.endswith("/chat/completions") or "/chat/completions?" in request.url.path:
+            endpoint_format = "openai"
+        elif request.url.path.endswith("/messages") or "/messages?" in request.url.path:
+            endpoint_format = "anthropic"
+        else:
+            endpoint_format = "unknown"
+
+        if endpoint_format == "unknown":
+            # Directly bypass the middleware
+            return await call_next(request)
+
+        # Read the request body
+        try:
+            json_body = await request.json()
+        except json.JSONDecodeError:
+            logger.warning(f"Request body is not valid JSON: {request.body}")
+            return await call_next(request)
+
+        # Check if streaming is requested
+        is_streaming = json_body.get("stream", False)
+
+        # Simple case: no streaming requested, just return the response
+        if not is_streaming:
+            return await call_next(request)
+
+        # Now the stream case
+        return await self._handle_stream_case(request, json_body, endpoint_format, call_next)
+
+    async def _handle_stream_case(
+        self,
+        request: Request,
+        json_body: Dict[str, Any],
+        endpoint_format: Literal["openai", "anthropic"],
+        call_next: Callable[[Request], Awaitable[Response]],
+    ) -> Response:
+        # 1) Modify the request body to force stream=False
+        modified_json = dict(json_body)
+        modified_json["stream"] = False
+        modified_body = json.dumps(modified_json).encode("utf-8")
+
+        # 2) Build a new scope + receive that yields our modified body
+        scope: Scope = dict(request.scope)
+        # rewrite headers for accept/content-length
+        new_headers: List[Tuple[bytes, bytes]] = []
+        saw_accept = False
+        for k, v in scope["headers"]:
+            kl = k.lower()
+            if kl == b"accept":
+                saw_accept = True
+                new_headers.append((k, b"application/json"))
+            elif kl == b"content-length":
+                # replace with new length
+                continue
+            else:
+                new_headers.append((k, v))
+        if not saw_accept:
+            new_headers.append((b"accept", b"application/json"))
+        new_headers.append((b"content-length", str(len(modified_body)).encode("ascii")))
+        scope["headers"] = new_headers
+
+        # Directly modify the request body
+        # Creating a new request won't work because request is cached in the base class
+        request._body = modified_body  # type: ignore
+
+        response = await call_next(request)
+
+        buffered: Optional[bytes] = None
+        # 4) If OK, buffer the response body (it should be JSON because we forced stream=False)
+        if 200 <= response.status_code < 300:
+            try:
+                if hasattr(response, "body_iterator"):
+                    # Buffer body safely
+                    body_chunks: List[bytes] = []
+                    async for chunk in response.body_iterator:  # type: ignore
+                        body_chunks.append(chunk)  # type: ignore
+                    buffered = b"".join(body_chunks)
+                else:
+                    buffered = response.body  # type: ignore
+
+                data = json.loads(buffered or b"{}")
+
+                if endpoint_format == "anthropic":
+                    return StreamingResponse(
+                        self.anthropic_stream_generator(data),
+                        media_type="text/event-stream",
+                        headers={"Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no"},
+                    )
+                else:
+                    # openai format
+                    return StreamingResponse(
+                        self.openai_stream_generator(data),
+                        media_type="text/event-stream",
+                        headers={"Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no"},
+                    )
+            except Exception as e:
+                # If anything goes wrong, fall back to non-streaming JSON
+                logger.exception(f"Error converting to stream; returning non-stream response: {e}")
+                # Rebuild the consumed response
+                return Response(
+                    content=buffered if buffered is not None else b"",
+                    status_code=response.status_code,
+                    headers=dict(response.headers),
+                    media_type=response.media_type,
+                    background=response.background,
+                )
+        else:
+            return response
+
+    async def anthropic_stream_generator(self, original_response: Dict[str, Any]):
+        """Generate Anthropic SSE-formatted chunks from complete content blocks
+
+        This is a dirty hack for Anthropic-style streaming from non-streaming response.
+        The sse format is subject to change based on Anthropic's implementation.
+        If so, try to use `MessageInspectionMiddleware` to inspect the update and fix accordingly.
+        """
+        # Anthropic format - handle multiple content blocks (text + tool_use)
+        content_blocks: List[Dict[str, Any]] = original_response.get("content", [])
+        message_id = original_response.get("id", f"msg_{int(time.time() * 1000)}")
+        model = original_response.get("model", "claude")
+
+        # Send message_start event
+        message_start: Dict[str, Any] = {
+            "type": "message_start",
+            "message": {
+                "id": message_id,
+                "type": "message",
+                "role": "assistant",
+                "content": [],
+                "model": model,
+                "stop_reason": None,
+                "stop_sequence": None,
+                "usage": original_response.get("usage", {"input_tokens": 0, "output_tokens": 0}),
+            },
+        }
+        yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n"
+
+        # Send ping to keep connection alive
+        ping = {"type": "ping"}
+        yield f"event: ping\ndata: {json.dumps(ping)}\n\n"
+
+        # Process each content block
+        for block_index, block in enumerate(content_blocks):
+            block_type = block.get("type", "text")
+
+            if block_type == "text":
+                # Handle text block
+                content = block.get("text", "")
+
+                # Send content_block_start event
+                content_block_start = {
+                    "type": "content_block_start",
+                    "index": block_index,
+                    "content_block": {"type": "text", "text": ""},
+                }
+                yield f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n"
+
+                # Stream text content in chunks
+                if content:
+                    words = content.split()
+                    chunk_size = 5
+
+                    for i in range(0, len(words), chunk_size):
+                        chunk_words = words[i : i + chunk_size]
+                        text_chunk = " ".join(chunk_words)
+
+                        # Add space after chunk unless it's the last one
+                        if i + chunk_size < len(words):
+                            text_chunk += " "
+
+                        content_block_delta = {
+                            "type": "content_block_delta",
+                            "index": block_index,
+                            "delta": {"type": "text_delta", "text": text_chunk},
+                        }
+                        yield f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n"
+                        await asyncio.sleep(0.02)
+
+                # Send content_block_stop event
+                content_block_stop = {"type": "content_block_stop", "index": block_index}
+                yield f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n"
+
+            elif block_type == "tool_use":
+                # Handle tool_use block
+                tool_name = block.get("name", "")
+                tool_input = block.get("input", {})
+                tool_id = block.get("id", f"toolu_{int(time.time() * 1000)}")
+
+                # Send content_block_start event for tool use
+                content_block_start: Dict[str, Any] = {
+                    "type": "content_block_start",
+                    "index": block_index,
+                    "content_block": {"type": "tool_use", "id": tool_id, "name": tool_name, "input": {}},
+                }
+                yield f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n"
+
+                # Stream tool input as JSON string chunks
+                input_json = json.dumps(tool_input)
+                chunk_size = 20  # characters per chunk for JSON
+
+                for i in range(0, len(input_json), chunk_size):
+                    json_chunk = input_json[i : i + chunk_size]
+
+                    content_block_delta = {
+                        "type": "content_block_delta",
+                        "index": block_index,
+                        "delta": {"type": "input_json_delta", "partial_json": json_chunk},
+                    }
+                    yield f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n"
+                    await asyncio.sleep(0.01)
+
+                # Send content_block_stop event
+                content_block_stop = {"type": "content_block_stop", "index": block_index}
+                yield f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n"
+
+        # Send message_delta event with stop reason
+        message_delta = {
+            "type": "message_delta",
+            "delta": {"stop_reason": original_response.get("stop_reason", "end_turn"), "stop_sequence": None},
+            "usage": {"output_tokens": original_response.get("usage", {}).get("output_tokens", 0)},
+        }
+        yield f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n"
+
+        # Send message_stop event
+        message_stop = {"type": "message_stop"}
+        yield f"event: message_stop\ndata: {json.dumps(message_stop)}\n\n"
+
+    async def openai_stream_generator(self, response_json: Dict[str, Any]) -> AsyncGenerator[str, Any]:
+        """
+        Convert a *complete* OpenAI chat.completions choice into a stream of
+        OpenAI-compatible SSE chunks.
+
+        This emits:
+
+          - an initial delta with the role ("assistant"),
+          - a sequence of deltas for message.content (split into small chunks),
+          - deltas for any tool_calls (including id/name and chunked arguments),
+          - a terminal chunk with finish_reason,
+          - and finally the literal '[DONE]'.
+
+        Notes:
+
+        - We only handle a *single* choice (index 0 typically).
+        - We purposefully don't attempt to stream logprobs.
+        - Chunking strategy is simple and conservative to avoid splitting
+          multi-byte characters: we slice on spaces where possible, then fall
+          back to fixed-size substrings.
+        """
+        choice = cast(Dict[str, Any], (response_json.get("choices") or [{}])[0])
+        model = response_json.get("model", "unknown")
+        created: int = int(time.time())
+        index: int = choice.get("index", 0)
+
+        message: Dict[str, Any] = choice.get("message", {}) or {}
+        role: str = message.get("role", "assistant")
+        content: str = message.get("content") or ""
+        tool_calls: List[Any] = message.get("tool_calls") or []
+        finish_reason: Optional[str] = choice.get(
+            "finish_reason"
+        )  # e.g., "stop", "length", "tool_calls", "content_filter"
+
+        def sse_chunk(obj: Dict[str, Any]) -> str:
+            return f"data: {json.dumps(obj, ensure_ascii=False)}\n\n"
+
+        # 1) initial chunk with the role
+        yield sse_chunk(
+            {
+                "id": f"chatcmpl-{created}",
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": model,
+                "choices": [{"index": index, "delta": {"role": role}, "finish_reason": None}],
+            }
+        )
+
+        # 2) stream textual content as small deltas
+        async def stream_content(text: str):
+            if not text:
+                return
+            # prefer splitting on spaces in ~20–40 char pieces
+            approx = 28
+            start = 0
+            n = len(text)
+            while start < n:
+                end = min(start + approx, n)
+                if end < n:
+                    # try to break on a space going forward
+                    space = text.rfind(" ", start, end)
+                    if space > start:
+                        end = space + 1
+                delta_text = text[start:end]
+                start = end
+                if not delta_text:
+                    break
+                yield sse_chunk(
+                    {
+                        "id": f"chatcmpl-{created}",
+                        "object": "chat.completion.chunk",
+                        "created": created,
+                        "model": model,
+                        "choices": [{"index": index, "delta": {"content": delta_text}, "finish_reason": None}],
+                    }
+                )
+                # tiny pause helps some UIs animate smoothly; keep very small
+                await asyncio.sleep(0.0)
+
+        async for piece in stream_content(content):  # type: ignore[misc]
+            yield piece  # pass through the produced chunks
+
+        # 3) stream tool_calls if present (id/name first, then arguments piecemeal)
+        for tc_index, tc in enumerate(tool_calls):
+            tc_type = tc.get("type", "function")
+            tc_id = tc.get("id") or f"call_{created}_{tc_index}"
+            fn: Dict[str, Any] = (tc.get("function") or {}) if tc_type == "function" else {}
+            fn_name: str = fn.get("name", "")
+            fn_args: str = fn.get("arguments", "") or ""
+
+            # (a) delta that announces the tool call id/type/name
+            yield sse_chunk(
+                {
+                    "id": f"chatcmpl-{created}",
+                    "object": "chat.completion.chunk",
+                    "created": created,
+                    "model": model,
+                    "choices": [
+                        {
+                            "index": index,
+                            "delta": {
+                                "tool_calls": [
+                                    {"index": tc_index, "id": tc_id, "type": tc_type, "function": {"name": fn_name}}
+                                ]
+                            },
+                            "finish_reason": None,
+                        }
+                    ],
+                }
+            )
+
+            # (b) stream arguments in small substrings
+            arg_chunk_size = 40
+            for pos in range(0, len(fn_args), arg_chunk_size):
+                partial = fn_args[pos : pos + arg_chunk_size]
+                yield sse_chunk(
+                    {
+                        "id": f"chatcmpl-{created}",
+                        "object": "chat.completion.chunk",
+                        "created": created,
+                        "model": model,
+                        "choices": [
+                            {
+                                "index": index,
+                                "delta": {"tool_calls": [{"index": tc_index, "function": {"arguments": partial}}]},
+                                "finish_reason": None,
+                            }
+                        ],
+                    }
+                )
+                await asyncio.sleep(0.0)
+
+        # 4) terminal chunk with finish_reason (default to "stop" if missing)
+        yield sse_chunk(
+            {
+                "id": f"chatcmpl-{created}",
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": model,
+                "choices": [
+                    {
+                        "index": index,
+                        "delta": {},
+                        "finish_reason": finish_reason or ("tool_calls" if tool_calls else "stop"),
+                    }
+                ],
+            }
+        )
+
+        # 5) literal DONE sentinel
+        yield "data: [DONE]\n\n"
+
+
+_MIDDLEWARE_REGISTRY: Dict[str, Type[BaseHTTPMiddleware]] = {
+    "rollout_attempt": RolloutAttemptMiddleware,
+    "stream_conversion": StreamConversionMiddleware,
+    "message_inspection": MessageInspectionMiddleware,
+}
+
+
+_CALLBACK_REGISTRY = {
+    "return_token_ids": AddReturnTokenIds,
+    "logprobs": AddLogprobs,
+    "opentelemetry": LightningOpenTelemetry,
+}
+
+
 class LLMProxy:
     """Host a LiteLLM OpenAI-compatible proxy bound to a LightningStore.
 
@@ -522,7 +1027,10 @@ class LLMProxy:
 
     !!! warning
 
-        The LLM Proxy does support streaming, but the tracing is still problematic when streaming is enabled.
+        By default (or when "stream_conversion" middleware is enabled), the LLM Proxy will convert OpenAI and Anthropic requests with `stream=True`
+        to a non-streaming request before going through the LiteLLM proxy. This is because the OpenTelemetry tracer provided by
+        LiteLLM is buggy with streaming responses. You can disable this by removing the "stream_conversion" middleware.
+        In that case, you might lose some tracing information like token IDs.
 
     !!! danger
 
@@ -544,6 +1052,13 @@ class LLMProxy:
             `launch_mode="asyncio"` launches the server in the current thread as an asyncio task.
             It is NOT recommended because it often causes hanging requests. Only use it if you know what you are doing.
         launcher_args: Arguments for the server launcher. If this is provided, host, port, and launch_mode will be ignored. Cannot be used together with port, host, and launch_mode.
+        middlewares: List of FastAPI middleware classes or strings to register. You can specify the class aliases or classes that have been imported.
+            If not provided, the default middlewares (RolloutAttemptMiddleware and StreamConversionMiddleware) will be used.
+            Available middleware aliases are: "rollout_attempt", "stream_conversion", "message_inspection".
+            Middlewares are the **first layer** of request processing. They are applied to all requests before the LiteLLM proxy.
+        callbacks: List of LiteLLM callback classes or strings to register. You can specify the class aliases or classes that have been imported.
+            If not provided, the default callbacks (AddReturnTokenIds and LightningOpenTelemetry) will be used.
+            Available callback aliases are: "return_token_ids", "opentelemetry", "logprobs".
     """
 
     def __init__(
@@ -557,7 +1072,8 @@ class LLMProxy:
         num_workers: int = 1,
         launch_mode: LaunchMode = "mp",
         launcher_args: PythonServerLauncherArgs | None = None,
-        _add_return_token_ids: bool = True,
+        middlewares: Sequence[Union[Type[BaseHTTPMiddleware], str]] | None = None,
+        callbacks: Sequence[Union[Type[CustomLogger], str]] | None = None,
     ):
         self.store = store
 
@@ -589,7 +1105,33 @@ class LLMProxy:
 
         self._config_file = None
 
-        self._add_return_token_ids = _add_return_token_ids
+        self.middlewares: List[Type[BaseHTTPMiddleware]] = []
+        if middlewares is None:
+            middlewares = ["rollout_attempt", "stream_conversion"]
+        for middleware in middlewares:
+            if isinstance(middleware, str):
+                if middleware not in _MIDDLEWARE_REGISTRY:
+                    raise ValueError(
+                        f"Invalid middleware alias: {middleware}. Available aliases are: {list(_MIDDLEWARE_REGISTRY.keys())}"
+                    )
+                middleware = _MIDDLEWARE_REGISTRY[middleware]
+                self.middlewares.append(middleware)
+            else:
+                self.middlewares.append(middleware)
+
+        self.callbacks: List[Type[CustomLogger]] = []
+        if callbacks is None:
+            callbacks = ["return_token_ids", "opentelemetry"]
+        for callback in callbacks:
+            if isinstance(callback, str):
+                if callback not in _CALLBACK_REGISTRY:
+                    raise ValueError(
+                        f"Invalid callback alias: {callback}. Available aliases are: {list(_CALLBACK_REGISTRY.keys())}"
+                    )
+                callback = _CALLBACK_REGISTRY[callback]
+                self.callbacks.append(callback)
+            else:
+                self.callbacks.append(callback)
 
     def get_store(self) -> Optional[LightningStore]:
         """Get the store used by the proxy.
@@ -637,25 +1179,25 @@ class LLMProxy:
         if _global_llm_proxy is not None:
             logger.warning("A global LLMProxy is already set. Overwriting it with the new instance.")
 
+        # Patch for LiteLLM v1.80.6+: https://github.com/BerriAI/litellm/issues/17243
+        os.environ["USE_OTEL_LITELLM_REQUEST_SPAN"] = "true"
+
         # Set the global LLMProxy reference for middleware/exporter access.
         set_active_llm_proxy(self)
 
         # Install middleware if it's not already installed.
-        installed: bool = False
+        installation_status: Dict[Any, bool] = {}
         for mw in app.user_middleware:
-            if mw.cls is RolloutAttemptMiddleware:
-                # Check whether the middleware is installed.
-                # It could be installed by other LLM Proxy instances, but it doesn't matter.
-                logger.info("Found existing RolloutAttemptMiddleware installed. Will not install a new one.")
-                installed = True
-                break
+            installation_status[mw.cls] = True
 
-        if not installed:
-            # Fallback to adding a new middleware
-            logger.info("Adding a new middleware to the FastAPI app.")
-            app.add_middleware(RolloutAttemptMiddleware)
+        for mw in self.middlewares:
+            if mw not in installation_status:
+                logger.info(f"Adding middleware {mw} to the FastAPI app.")
+                app.add_middleware(mw)
+            else:
+                logger.info(f"Middleware {mw} is already installed. Will not install a new one.")
 
-        if not initialize_llm_callbacks(self._add_return_token_ids):
+        if not initialize_llm_callbacks(self.callbacks):
             # If it's not the first time to initialize the callbacks, also
             # reset LiteLLM's logging worker so its asyncio.Queue binds to the new loop.
             _reset_litellm_logging_worker()
@@ -720,16 +1262,16 @@ class LLMProxy:
         if self.store is None:
             raise ValueError("Store is not set. Please set the store before starting the LLMProxy.")
 
-        store_capabilities = self.store.capabilities()
-        if self.server_launcher.args.launch_mode == "mp" and not store_capabilities["zero_copy"]:
+        store_capabilities = self.store.capabilities
+        if self.server_launcher.args.launch_mode == "mp" and not store_capabilities.get("zero_copy", False):
             raise RuntimeError(
                 "The store does not support zero-copy. Please use another store, or use asyncio or thread mode to launch the server."
             )
-        elif self.server_launcher.args.launch_mode == "thread" and not store_capabilities["thread_safe"]:
+        elif self.server_launcher.args.launch_mode == "thread" and not store_capabilities.get("thread_safe", False):
             raise RuntimeError(
                 "The store is not thread-safe. Please use another store, or use asyncio mode to launch the server."
             )
-        elif self.server_launcher.args.launch_mode == "asyncio" and not store_capabilities["async_safe"]:
+        elif self.server_launcher.args.launch_mode == "asyncio" and not store_capabilities.get("async_safe", False):
             raise RuntimeError("The store is not async-safe. Please use another store.")
 
         logger.info(
@@ -849,7 +1391,7 @@ def set_active_llm_proxy(proxy: LLMProxy) -> None:
     _global_llm_proxy = proxy
 
 
-def initialize_llm_callbacks(_add_return_token_ids: bool = True) -> bool:
+def initialize_llm_callbacks(callback_classes: List[Type[CustomLogger]]) -> bool:
     """Restore `litellm.callbacks` to a state that is just initialized by agent-lightning.
 
     When litellm is restarted multiple times in the same process, more and more callbacks
@@ -857,8 +1399,7 @@ def initialize_llm_callbacks(_add_return_token_ids: bool = True) -> bool:
     This function remembers the initial state of `litellm.callbacks` and always restore to that state.
 
     Args:
-        _add_return_token_ids: Whether to add the return token ids callback. Internal use only.
-        Ideally the callback should automatically be enabled when the backend supports it.
+        callback_classes: List of callback classes to register.
 
     Returns:
         Whether the callbacks are initialized for the first time.
@@ -866,31 +1407,31 @@ def initialize_llm_callbacks(_add_return_token_ids: bool = True) -> bool:
     global _callbacks_before_litellm_start
 
     if _callbacks_before_litellm_start is None:
-        litellm.callbacks.extend(  # type: ignore
-            [
-                AddReturnTokenIds(),
-                LightningOpenTelemetry(),
-            ]
-            if _add_return_token_ids
-            else [
-                LightningOpenTelemetry(),
-            ]
-        )
+        litellm.callbacks.extend([cls() for cls in callback_classes])  # type: ignore
         _callbacks_before_litellm_start = [*litellm.callbacks]  # type: ignore
         return True
 
+    else:
+        # Put whatever is missing in the new callback classes to the existing callbacks.
+        for cls in callback_classes:
+            if not any(isinstance(cb, cls) for cb in _callbacks_before_litellm_start):
+                logger.info(f"Adding missing callback {cls} to the existing callbacks.")
+                _callbacks_before_litellm_start.append(cls())
+
     _reset_litellm_logging_callback_manager()
 
-    # Check if tracer provider is malformed due to global tracer clear in tests.
-    if not _check_tracer_provider():
-        logger.warning(
-            "Global tracer provider might have been cleared outside. Re-initializing OpenTelemetry callback."
-        )
-        _callbacks_before_litellm_start = [
-            cb for cb in _callbacks_before_litellm_start if not isinstance(cb, LightningOpenTelemetry)
-        ] + [LightningOpenTelemetry()]
-    else:
-        logger.debug("Global tracer provider is valid. Reusing existing OpenTelemetry callback.")
+    if LightningOpenTelemetry in callback_classes:
+        # Check if tracer provider is malformed due to global tracer clear in tests.
+        if not _check_tracer_provider():
+            logger.warning(
+                "Global tracer provider might have been cleared outside. Re-initializing OpenTelemetry callback."
+            )
+            _callbacks_before_litellm_start = [
+                cb for cb in _callbacks_before_litellm_start if not isinstance(cb, LightningOpenTelemetry)
+            ] + [LightningOpenTelemetry()]
+        else:
+            logger.debug("Global tracer provider is valid. Reusing existing OpenTelemetry callback.")
+    # Otherwise, we just skip the check for opentelemetry and use the existing callback.
 
     litellm.callbacks.clear()  # type: ignore
     litellm.callbacks.extend(_callbacks_before_litellm_start)  # type: ignore

agentlightning/logging.py

@@ -1,10 +1,18 @@
 # Copyright (c) Microsoft. All rights reserved.
 
+from __future__ import annotations
+
 import logging
 import os
 import platform
+import sys
+import warnings
+from logging.config import dictConfig
+from typing import Any, Dict, Optional
 
-__all__ = ["configure_logger"]
+from rich.console import Console
+
+__all__ = ["setup", "configure_logger", "setup_module"]
 
 
 def configure_logger(level: int = logging.INFO, name: str = "agentlightning") -> logging.Logger:
@@ -15,6 +23,10 @@ def configure_logger(level: int = logging.INFO, name: str = "agentlightning") ->
     not propagate to the root logger, preventing duplicate log emission when
     applications compose multiple logging configurations.
 
+    !!! danger
+
+        This function is deprecated in favor of [`setup_logging`][agentlightning.setup_logging].
+
     Args:
         level: Logging level applied both to the logger and the installed
             handler. Defaults to `logging.INFO`.
@@ -32,23 +44,327 @@ def configure_logger(level: int = logging.INFO, name: str = "agentlightning") ->
         logger.info("agent-lightning is ready!")
         ```
     """
+    warnings.warn("This function is deprecated in favor of `setup_logging`.", DeprecationWarning, stacklevel=2)
 
+    return setup_module(level=level, name=name, console=True, color=True, propagate=False)
+
+
+DEFAULT_FORMAT = "%(asctime)s [%(levelname)s] (Process-%(process)d %(name)s)   %(message)s"
+DATE_FORMAT = "%H:%M:%S"
+
+
+def _to_level_value(lvl: int | str) -> int:
+    if isinstance(lvl, int):
+        return lvl
+    val = getattr(logging, str(lvl).upper(), None)
+    if val is None:
+        raise ValueError(f"Invalid log level: {lvl}")
+    return val
+
+
+def _ensure_file_handler(
+    logger: logging.Logger,
+    filename: str,
+    *,
+    level: int,
+    formatter: Optional[logging.Formatter],
+) -> None:
+    """Attach a FileHandler to `logger` for `filename` if it doesn't already exist."""
+    abspath = os.path.abspath(filename)
+
+    # Avoid duplicates
+    for h in logger.handlers:
+        if isinstance(h, logging.FileHandler) and getattr(h, "baseFilename", None) == abspath:
+            return
+
+    # Ensure directory exists
+    dirname = os.path.dirname(abspath)
+    if dirname:
+        os.makedirs(dirname, exist_ok=True)
+
+    fh = logging.FileHandler(abspath, encoding="utf-8")
+    fh.setLevel(level)
+    if formatter is not None:
+        fh.setFormatter(formatter)
+    else:
+        fh.setFormatter(logging.Formatter(DEFAULT_FORMAT, DATE_FORMAT))
+
+    logger.addHandler(fh)
+
+
+def setup(
+    level: int | str = "INFO",
+    *,
+    console: bool = True,
+    color: bool | Dict[str, Any] = True,
+    propagate: bool = False,
+    disable_existing_loggers: bool = False,
+    capture_warnings: bool = False,
+    submodule_levels: Optional[dict[str, int | str]] = None,
+    extra_handlers: Optional[list[logging.Handler]] = None,
+    formatter: Optional[logging.Formatter] = None,
+    apply_to: Optional[list[str]] = None,
+    files: Optional[str | dict[str, str]] = None,
+) -> None:
+    """Configures logging for the `agentlightning` logger hierarchy.
+
+    This function provides a one-stop setup utility for configuring the
+    `agentlightning` root logger and optionally its submodules or external
+    loggers. It supports console logging, colored rich output, per-submodule
+    log levels, and optional handler/formatter injection.
+
+    The setup is intentionally isolated: it does not modify the global root
+    logger or loggers belonging to other libraries unless explicitly directed
+    via `apply_to`.
+
+    Args:
+        level:
+            Logging level for the base `agentlightning` logger. Accepts either
+            an integer (e.g., `logging.DEBUG`) or a string level name
+            (e.g., `"INFO"`). Defaults to `"INFO"`.
+        console:
+            Whether to attach a console handler to the logger. Defaults to
+            `True`.
+        color:
+            Enables rich-formatted output using `RichHandler` when `True`
+            or a configuration dict. If `False`, a plain text formatter is
+            used instead. Defaults to `True`.
+        propagate:
+            Whether `agentlightning` logs should propagate to ancestor
+            loggers. Defaults to `False`.
+        disable_existing_loggers:
+            Passed to `logging.config.dictConfig`. If `True`, disables all
+            existing configured loggers before applying this configuration.
+            Defaults to `False`.
+        capture_warnings:
+            If `True`, redirects Python `warnings` emitted via the `warnings`
+            module into the logging system. Defaults to `False`.
+        submodule_levels:
+            Mapping of submodule logger names to logging levels. If a specified
+            submodule level is more verbose than the base level, a warning is emitted.
+        extra_handlers:
+            A list of user-provided handlers to attach to the `agentlightning` logger.
+            Handlers are added idempotently; duplicates are not reattached.
+        formatter:
+            A formatter to apply to any handler under `agentlightning` that does not
+            already have one assigned. Useful for customizing output without overwriting
+            formatters on custom handlers.
+        apply_to:
+            A list of additional logger names to configure identically to
+            `agentlightning` base logger. Their handlers are replaced with copies of the base
+            handlers, and propagation is disabled to avoid duplicate log emission.
+        files:
+            If a string, attach a FileHandler to the base `agentlightning` logger.
+            If a dict, for each `(logger_name, filename)` pair, attach a FileHandler
+            directly to that logger.
+            Each file handler should use the logger's effective level at creation.
+
+    Notes:
+        * On Windows, this function forces UTF-8 mode in the console to prevent
+          issues with rich output or special characters.
+        * Submodule loggers can generate records below the handler's emission
+          threshold. Whether such records appear depends on both the logger's
+          level and the handler's level.
+        * `apply_to` loggers inherit the same handlers but do not propagate
+          upward, yielding isolated, consistent behavior.
+
+    Examples:
+        Basic setup:
+
+        >>> setup()
+
+        Enabling debug mode with no color:
+
+        >>> setup(level="DEBUG", color=False)
+
+        Overriding specific submodule levels:
+
+        >>> setup(submodule_levels={"agentlightning.io": "DEBUG"})
+
+        Attaching an additional file handler:
+
+        >>> fh = logging.FileHandler("app.log")
+        >>> setup(extra_handlers=[fh])
+    """
     # Ensure UTF-8 encoding on Windows consoles
-    # Note: This change does not fully represent support for execution under the windown system.
+    # Note: This change does not fully represent support for execution under the windows system.
     # It only fixes console printing issues caused by special characters.
     # TODO: More comprehensive Windows support may be needed in the future.
     if platform.system() == "Windows":
         os.environ["PYTHONUTF8"] = "1"
 
-    logger = logging.getLogger(name)
-    logger.handlers.clear()  # clear existing handlers
+    base_logger = setup_module(
+        level,
+        name="agentlightning",
+        console=console,
+        color=color,
+        propagate=propagate,
+        disable_existing_loggers=disable_existing_loggers,
+    )
 
-    # log to stdout
-    handler = logging.StreamHandler()
-    handler.setLevel(level)
-    formatter = logging.Formatter("%(asctime)s [%(levelname)s] (Process-%(process)d %(name)s)   %(message)s")
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    logger.setLevel(level)
-    logger.propagate = False  # prevent double logging
-    return logger
+    base_level_value = base_logger.level
+
+    # Apply user-provided formatter (only to handlers without one,
+    # so we don't clobber custom extra_handlers)
+    if formatter is not None:
+        for h in base_logger.handlers:
+            if h.formatter is None:
+                h.setFormatter(formatter)
+
+    # Attach user-provided handler(s) if any, idempotently
+    if extra_handlers:
+        for h in extra_handlers:
+            if h not in base_logger.handlers:
+                base_logger.addHandler(h)
+
+    # Per-submodule levels
+    if submodule_levels:
+        for name, lvl in submodule_levels.items():
+            sub_level = _to_level_value(lvl)
+
+            # Emit a warning if submodule level is lower (more verbose) than the global/base level
+            if sub_level < base_level_value:
+                base_logger.warning(
+                    "Submodule logger '%s' level %s (%s) is more verbose than base "
+                    "logger level %s (%s). Records below the base level may still be "
+                    "filtered out by handlers depending on their own levels.",
+                    name,
+                    lvl,
+                    sub_level,
+                    logging.getLevelName(base_level_value),
+                    base_level_value,
+                )
+
+            # The logger will *create* records down to the logger's level, but a handler
+            # with a higher level will still drop anything below its own threshold.
+            # Effective emission is gated by both: record.level >= logger.level AND handler.level.
+            logging.getLogger(name).setLevel(lvl)
+
+    # Attach file handlers if requested
+    if files is not None:
+        if isinstance(files, str):
+            # Single file for the entire `agentlightning` hierarchy.
+            _ensure_file_handler(
+                logger=base_logger,
+                filename=files,
+                level=base_level_value,
+                formatter=formatter,
+            )
+        else:
+            # Per-logger files
+            for logger_name, filename in files.items():
+                lg = logging.getLogger(logger_name)
+                # Use the logger's *effective* level at creation time
+                effective_level = lg.getEffectiveLevel()
+                _ensure_file_handler(
+                    logger=lg,
+                    filename=filename,
+                    level=effective_level,
+                    formatter=formatter,
+                )
+
+    # Optionally apply the same handler setup to other loggers outside this module
+    if apply_to:
+        for name in apply_to:
+            lg = logging.getLogger(name)
+            # This removes any existing handlers so we don't duplicate output
+            # and ensures these loggers share exactly the same handlers as base_logger.
+            lg.handlers.clear()
+            for h in base_logger.handlers:
+                lg.addHandler(h)
+            lg.setLevel(base_logger.level)
+            # We've attached handlers directly to these loggers; if propagate
+            # stayed True, records would bubble up to ancestor loggers and could be
+            # emitted twice (here and on the parent/root). Setting False isolates them.
+            lg.propagate = False
+
+    # Optionally capture warnings
+    if capture_warnings:
+        logging.captureWarnings(True)
+
+
+def setup_module(
+    level: int | str = "INFO",
+    *,
+    name: str = "agentlightning",
+    console: bool = True,
+    color: bool | Dict[str, Any] = True,
+    propagate: bool = False,
+    disable_existing_loggers: bool = False,
+) -> logging.Logger:
+    """Initializes and returns the base logger for `agentlightning`.
+
+    This function constructs and applies a `dictConfig` configuration for the
+    logger hierarchy rooted at `name`. It supports either rich console
+    formatting (via `RichHandler`) or plain text formatting, based on the
+    `color` argument.
+
+    Unlike [`setup_logging`][agentlightning.setup_logging], this function configures only a single logger namespace
+    and does not attach extra handlers or submodule levels. It is primarily used
+    internally by [`setup_logging`][agentlightning.setup_logging] but is also suitable for direct integration in
+    custom logging workflows.
+    """
+    root_cfg: Dict[str, Any] = {
+        "version": 1,
+        "disable_existing_loggers": disable_existing_loggers,
+        "loggers": {
+            name: {
+                "handlers": [],
+                "level": level,
+                "propagate": propagate,
+            }
+        },
+        "handlers": {},
+        "formatters": {},
+    }
+
+    # Choose formatter / handler definition
+    if color is not False and console:
+        # Console must be true to display colored outputs
+        if isinstance(color, dict):
+            rich_handler_config = color
+        else:
+            rich_handler_config: Dict[str, Any] = {
+                "rich_tracebacks": False,
+                "markup": False,
+                "show_time": True,
+                "show_path": True,
+            }
+
+            if not _has_width():
+                # e.g., in a CI environment.
+                rich_handler_config["console"] = Console(width=200)
+
+        root_cfg["handlers"]["console"] = {
+            "class": "rich.logging.RichHandler",
+            "level": level,
+            **rich_handler_config,
+        }
+        # RichHandler manages its own style; keep formatter None
+    else:
+        fmt_name = "plain"
+        root_cfg["formatters"][fmt_name] = {
+            "format": DEFAULT_FORMAT,
+            "datefmt": DATE_FORMAT,
+        }
+
+        if console:
+            root_cfg["handlers"]["console"] = {
+                "class": "logging.StreamHandler",
+                "level": level,
+                "formatter": fmt_name,
+            }
+
+    # Attach selected handlers to agentlightning
+    handler_names = list(root_cfg["handlers"].keys())
+    root_cfg["loggers"][name]["handlers"] = handler_names
+
+    # Apply dictConfig (this resets the logger handlers)
+    dictConfig(root_cfg)
+
+    return logging.getLogger(name)
+
+
+def _has_width() -> bool:
+    """Automatically determine whether the terminal has a width."""
+    return sys.stdout.isatty()

agentlightning/runner/agent.py

@@ -11,16 +11,30 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import random
+import threading
 import time
-from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, TypeVar, cast
+from contextlib import suppress
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    TypeVar,
+    cast,
+)
 
 from opentelemetry.sdk.trace import ReadableSpan
 
 from agentlightning.litagent import LitAgent
 from agentlightning.reward import emit_reward, find_final_reward
 from agentlightning.store.base import LightningStore
-from agentlightning.tracer.agentops import AgentOpsTracer
 from agentlightning.tracer.base import Tracer
+from agentlightning.tracer.otel import OtelTracer
 from agentlightning.types import (
     AttemptedRollout,
     Hook,
@@ -30,6 +44,7 @@ from agentlightning.types import (
     RolloutRawResult,
     Span,
 )
+from agentlightning.utils.system_snapshot import system_snapshot
 
 if TYPE_CHECKING:
     from agentlightning.execution.events import ExecutionEvent
@@ -52,7 +67,15 @@ class LitAgentRunner(Runner[T_task]):
         worker_id: Identifier for the active worker process, if any.
     """
 
-    def __init__(self, tracer: Tracer, max_rollouts: Optional[int] = None, poll_interval: float = 5.0) -> None:
+    def __init__(
+        self,
+        tracer: Tracer,
+        max_rollouts: Optional[int] = None,
+        poll_interval: float = 5.0,
+        heartbeat_interval: float = 10.0,
+        interval_jitter: float = 0.5,
+        heartbeat_launch_mode: Literal["asyncio", "thread"] = "asyncio",
+    ) -> None:
         """Initialize the agent runner.
 
         Args:
@@ -60,11 +83,21 @@ class LitAgentRunner(Runner[T_task]):
             max_rollouts: Optional cap on iterations processed by
                 [`iter`][agentlightning.LitAgentRunner.iter].
             poll_interval: Seconds to wait between store polls when no work is available.
+            heartbeat_interval: Seconds to wait between sending heartbeats to the store.
+            interval_jitter: Jitter factor for the poll interval. The actual interval will be between
+                poll_interval - interval_jitter and poll_interval + interval_jitter.
+                This is to avoid the overload caused by the synchronization of the runners.
+            heartbeat_launch_mode: Launch mode for the heartbeat loop. Can be "asyncio" or "thread".
+                "asyncio" is the default and recommended mode. Use "thread" if you are experiencing blocking coroutines.
         """
         super().__init__()
         self._tracer = tracer
         self._max_rollouts = max_rollouts
         self._poll_interval = poll_interval
+        self._heartbeat_interval = heartbeat_interval
+        self._interval_jitter = interval_jitter
+        self._heartbeat_launch_mode = heartbeat_launch_mode
+        self._random_state = random.Random()
 
         # Set later
         self._agent: Optional[LitAgent[T_task]] = None
@@ -105,7 +138,7 @@ class LitAgentRunner(Runner[T_task]):
         self._store = store
         self.worker_id = worker_id
 
-        self._tracer.init_worker(worker_id)
+        self._tracer.init_worker(worker_id, store)
 
     def teardown(self, *args: Any, **kwargs: Any) -> None:
         """Teardown the runner and clean up all resources.
@@ -244,20 +277,31 @@ class LitAgentRunner(Runner[T_task]):
         store = self.get_store()
 
         trace_spans: list[ReadableSpan] | list[Span] = []
+        result_recognized: bool = False
 
         # Case 0: result is None
         if raw_result is None:
             trace_spans = self._tracer.get_last_trace()
+            result_recognized = True
 
         # Case 1: result is a float (final reward)
-        if isinstance(raw_result, float):
+        if isinstance(raw_result, (bool, int, float)):
+            if isinstance(raw_result, (bool, int)):
+                logger.warning(
+                    f"{self._log_prefix(rollout.rollout_id)} Reward is not a number, got: {type(raw_result)}. "
+                    "Auto converting to float."
+                )
+                raw_result = float(raw_result)
             # Preserve the existing spans before another span is emitted
             trace_spans = list(self._tracer.get_last_trace())
-            # This will emit another span to the tracer
-            reward_span = emit_reward(raw_result)
+            # This will NOT emit another span to the tracer
+            reward_span = emit_reward(raw_result, propagate=False)
+            # We add it to the store manually
             await store.add_otel_span(rollout.rollout_id, rollout.attempt.attempt_id, reward_span)
             trace_spans.append(reward_span)
+            result_recognized = True
 
+        # Case 2-3: result is a list
         if isinstance(raw_result, list):
             # For rollout methods that return a list, we assume that the returned spans
             # are the complete span set from the whole rollout
@@ -265,10 +309,7 @@ class LitAgentRunner(Runner[T_task]):
 
             # Case 2: result is a list of ReadableSpan (OpenTelemetry spans)
             if len(raw_result) > 0 and all(isinstance(t, ReadableSpan) for t in raw_result):
-
-                if not isinstance(
-                    self._tracer, AgentOpsTracer
-                ):  # TODO: this should be replaced with general OpenTelemetry tracer in next version
+                if not isinstance(self._tracer, OtelTracer):
                     for span in raw_result:
                         await store.add_otel_span(
                             rollout.rollout_id, rollout.attempt.attempt_id, cast(ReadableSpan, span)
@@ -279,6 +320,7 @@ class LitAgentRunner(Runner[T_task]):
                         "The traces should have already been added to the store. "
                         "No need to return anything from rollout."
                     )
+                result_recognized = True
 
             # Case 3: result is a list of Span (agentlightning spans)
             elif len(raw_result) > 0 and all(isinstance(t, Span) for t in raw_result):
@@ -286,6 +328,7 @@ class LitAgentRunner(Runner[T_task]):
                 for span in raw_result:
                     await store.add_span(cast(Span, span))
                 trace_spans = raw_result
+                result_recognized = True
 
             # Left over cases for list
             elif len(raw_result) == 0:
@@ -294,6 +337,7 @@ class LitAgentRunner(Runner[T_task]):
                     "Please check your rollout implementation."
                 )
                 trace_spans = raw_result
+                result_recognized = True
 
             else:
                 types = [type(t).__name__ for t in raw_result][:10]
@@ -302,8 +346,83 @@ class LitAgentRunner(Runner[T_task]):
                     f"but got: {', '.join(types)}..."
                 )
 
+        if not result_recognized:
+            raise TypeError(
+                f"Invalid raw result type. It's expected to be none, float, or a list of ReadableSpan or Span, "
+                f"but got: {type(raw_result).__name__}..."
+            )
+
         return trace_spans
 
+    async def _emit_heartbeat(self, store: LightningStore) -> None:
+        """Send a heartbeat tick to the store."""
+        worker_id = self.get_worker_id()
+
+        try:
+            await store.update_worker(worker_id, system_snapshot())
+        except asyncio.CancelledError:
+            # bypass the exception
+            raise
+        except Exception:
+            logger.exception("%s Unable to update worker heartbeat.", self._log_prefix())
+
+    def _start_heartbeat_loop(self, store: LightningStore) -> Optional[Callable[[], Awaitable[None]]]:
+        """Start a background heartbeat loop and return an async stopper."""
+
+        if self._heartbeat_interval <= 0:
+            return None
+
+        if self.worker_id is None:
+            logger.warning("%s Cannot start heartbeat loop without worker_id.", self._log_prefix())
+            return None
+
+        if self._heartbeat_launch_mode == "asyncio":
+            stop_event = asyncio.Event()
+
+            async def heartbeat_loop() -> None:
+                while not stop_event.is_set():
+                    await self._emit_heartbeat(store)
+                    with suppress(asyncio.TimeoutError):
+                        interval = self._heartbeat_interval + self._random_state.uniform(
+                            -self._interval_jitter, self._interval_jitter
+                        )
+                        interval = max(interval, 0.01)
+                        await asyncio.wait_for(stop_event.wait(), timeout=interval)
+
+            task = asyncio.create_task(heartbeat_loop(), name=f"{self.get_worker_id()}-heartbeat")
+
+            async def stop() -> None:
+                stop_event.set()
+                with suppress(asyncio.CancelledError):
+                    await task
+
+            return stop
+
+        if self._heartbeat_launch_mode == "thread":
+            stop_evt = threading.Event()
+
+            def thread_worker() -> None:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                while not stop_evt.is_set():
+                    loop.run_until_complete(self._emit_heartbeat(store))
+                    interval = self._heartbeat_interval + self._random_state.uniform(
+                        -self._interval_jitter, self._interval_jitter
+                    )
+                    interval = max(interval, 0.01)
+                    stop_evt.wait(interval)
+
+            thread = threading.Thread(target=thread_worker, name=f"{self.get_worker_id()}-heartbeat", daemon=True)
+            thread.start()
+
+            async def stop() -> None:
+                stop_evt.set()
+                await asyncio.to_thread(thread.join)
+
+            return stop
+
+        raise ValueError(f"Unsupported heartbeat launch mode: {self._heartbeat_launch_mode}")
+
     async def _sleep_until_next_poll(self, event: Optional[ExecutionEvent] = None) -> None:
         """Sleep until the next poll interval, with optional event-based interruption.
 
@@ -314,11 +433,13 @@ class LitAgentRunner(Runner[T_task]):
             event: Optional [`ExecutionEvent`][agentlightning.ExecutionEvent] object that can be used to interrupt the sleep.
                 If set during the sleep period, the method returns immediately.
         """
+        interval = self._poll_interval + self._random_state.uniform(-self._interval_jitter, self._interval_jitter)
+        interval = max(interval, 0.01)
         if event is None:
-            await asyncio.sleep(self._poll_interval)
+            await asyncio.sleep(interval)
             return
         current_time = time.time()
-        next_time = current_time + self._poll_interval
+        next_time = current_time + interval
         while time.time() < next_time:
             await asyncio.sleep(0.1)
             if event.is_set():
@@ -364,7 +485,7 @@ class LitAgentRunner(Runner[T_task]):
 
             start_time = time.time()
             async with self._tracer.trace_context(
-                name=rollout_id, store=store, rollout_id=rollout_id, attempt_id=next_rollout.attempt.attempt_id
+                name=rollout_id, rollout_id=rollout_id, attempt_id=next_rollout.attempt.attempt_id
             ):
                 await self._trigger_hooks(
                     hook_type="on_trace_start", agent=agent, runner=self, tracer=self._tracer, rollout=next_rollout
@@ -450,39 +571,39 @@ class LitAgentRunner(Runner[T_task]):
         logger.info(f"{self._log_prefix()} Started async rollouts (max: {self._max_rollouts or 'unlimited'}).")
         store = self.get_store()
 
-        while not (event is not None and event.is_set()) and (
-            self._max_rollouts is None or num_tasks_processed < self._max_rollouts
-        ):
-            # Retrieve the next rollout
-            next_rollout: Optional[Rollout] = None
-            while not (event is not None and event.is_set()):
-                logger.debug(f"{self._log_prefix()} Try to poll for next rollout.")
-                next_rollout = await store.dequeue_rollout()
+        stop_heartbeat = self._start_heartbeat_loop(store)
+
+        try:
+            while not (event is not None and event.is_set()) and (
+                self._max_rollouts is None or num_tasks_processed < self._max_rollouts
+            ):
+                # Retrieve the next rollout
+                next_rollout: Optional[Rollout] = None
+                while not (event is not None and event.is_set()):
+                    logger.debug(f"{self._log_prefix()} Try to poll for next rollout.")
+                    next_rollout = await store.dequeue_rollout(worker_id=self.get_worker_id())
+                    if next_rollout is None:
+                        logger.debug(
+                            f"{self._log_prefix()} No rollout to poll. Waiting for {self._poll_interval} seconds."
+                        )
+                        await self._sleep_until_next_poll(event)
+                    else:
+                        break
+
                 if next_rollout is None:
-                    logger.debug(f"{self._log_prefix()} No rollout to poll. Waiting for {self._poll_interval} seconds.")
-                    await self._sleep_until_next_poll(event)
-                else:
-                    break
+                    return
 
-            if next_rollout is None:
-                return
+                # Execute the step
+                await self._step_impl(next_rollout)
 
-            try:
-                # Claim the rollout but updating the current worker id
-                await store.update_attempt(
-                    next_rollout.rollout_id, next_rollout.attempt.attempt_id, worker_id=self.get_worker_id()
-                )
-            except Exception:
-                # This exception could happen if the rollout is dequeued and the other end died for some reason
-                logger.exception(f"{self._log_prefix()} Exception during update_attempt, giving up the rollout.")
-                continue
-
-            # Execute the step
-            await self._step_impl(next_rollout)
-
-            num_tasks_processed += 1
-            if num_tasks_processed % 10 == 0 or num_tasks_processed == 1:
-                logger.info(f"{self._log_prefix()} Progress: {num_tasks_processed}/{self._max_rollouts or 'unlimited'}")
+                num_tasks_processed += 1
+                if num_tasks_processed % 10 == 0 or num_tasks_processed == 1:
+                    logger.info(
+                        f"{self._log_prefix()} Progress: {num_tasks_processed}/{self._max_rollouts or 'unlimited'}"
+                    )
+        finally:
+            if stop_heartbeat is not None:
+                await stop_heartbeat()
 
         logger.info(f"{self._log_prefix()} Finished async rollouts. Processed {num_tasks_processed} tasks.")
 
@@ -525,7 +646,9 @@ class LitAgentRunner(Runner[T_task]):
         else:
             resources_id = None
 
-        attempted_rollout = await self.get_store().start_rollout(input=input, mode=mode, resources_id=resources_id)
+        attempted_rollout = await self.get_store().start_rollout(
+            input=input, mode=mode, resources_id=resources_id, worker_id=self.get_worker_id()
+        )
         rollout_id = await self._step_impl(attempted_rollout, raise_on_exception=True)
 
         completed_rollout = await store.get_rollout_by_id(rollout_id)

agentlightning/semconv.py

@@ -0,0 +1,158 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Semantic conventions for Agent-lightning spans.
+
+Conventions in this file are added on demand. We generally DO NOT add
+new semantic conventions unless it's absolutely needed for certain algorithms or scenarios.
+"""
+
+from enum import Enum
+
+from pydantic import BaseModel
+
+AGL_ANNOTATION = "agentlightning.annotation"
+"""Agent-lightning's standard span name for annotations.
+
+Annotations are minimal span units for rewards, tags, and metadatas.
+They are used to "annotate" a specific event or a part of rollout.
+"""
+
+AGL_MESSAGE = "agentlightning.message"
+"""Agent-lightning's standard span name for messages and logs."""
+
+AGL_OBJECT = "agentlightning.object"
+"""Agent-lightning's standard span name for customized objects."""
+
+AGL_EXCEPTION = "agentlightning.exception"
+"""Agent-lightning's standard span name for exceptions.
+
+Used by the exception emitter to record exception details.
+"""
+
+AGL_OPERATION = "agentlightning.operation"
+"""Agent-lightning's standard span name for functions.
+Wrap function or code-blocks as operations.
+"""
+
+AGL_VIRTUAL = "agentlightning.virtual"
+"""Agent-lightning's standard span name for virtual operations.
+
+Mostly used in adapter when needing to represent the root or intermediate operations.
+"""
+
+
+class LightningResourceAttributes(Enum):
+    """Resource attribute names used in Agent-lightning spans."""
+
+    ROLLOUT_ID = "agentlightning.rollout_id"
+    """Resource name for rollout ID in Agent-lightning spans."""
+
+    ATTEMPT_ID = "agentlightning.attempt_id"
+    """Resource name for attempt ID in Agent-lightning spans."""
+
+    SPAN_SEQUENCE_ID = "agentlightning.span_sequence_id"
+    """Resource name for span sequence ID in Agent-lightning spans."""
+
+
+class LightningSpanAttributes(Enum):
+    """Attribute names that commonly appear in Agent-lightning spans.
+
+    Exception types can't be found here because they are defined in OpenTelemetry's official semantic conventions.
+    """
+
+    REWARD = "agentlightning.reward"
+    """Attribute prefix for rewards-related data in reward spans.
+
+    It should be used as a prefix. For example, "agentlightning.reward.0.value" can
+    be used to track a specific metric. See [RewardAttributes][agentlightning.semconv.RewardAttributes].
+    """
+
+    LINK = "agentlightning.link"
+    """Attribute name for linking the current span to another span or other objects like requests/responses."""
+
+    TAG = "agentlightning.tag"
+    """Attribute name for tagging spans with customized strings."""
+
+    MESSAGE_BODY = "agentlightning.message.body"
+    """Attribute name for message text in message spans."""
+
+    OBJECT_TYPE = "agentlightning.object.type"
+    """Attribute name for object type (full qualified name) in object spans.
+
+    I think builtin types like str, int, bool, list, dict are self-explanatory and
+    should also be qualified to use here.
+    """
+
+    OBJECT_LITERAL = "agentlightning.object.literal"
+    """Attribute name for object literal value in object spans (for str, int, bool, ...)."""
+
+    OBJECT_JSON = "agentlightning.object.json"
+    """Attribute name for object serialized value (JSON) in object spans."""
+
+    OPERATION_NAME = "agentlightning.operation.name"
+    """Attribute name for operation name in operation spans, normally the function name."""
+
+    OPERATION_INPUT = "agentlightning.operation.input"
+    """Attribute name for operation input in operation spans."""
+
+    OPERATION_OUTPUT = "agentlightning.operation.output"
+    """Attribute name for operation output in operation spans."""
+
+
+class RewardAttributes(Enum):
+    """Multi-dimensional reward attributes will look like:
+
+    ```json
+    {"agentlightning.reward.0.name": "efficiency", "agentlightning.reward.0.value": 0.75}
+    ```
+
+    The first reward in the reward list will automatically be the primary reward.
+    If the reward list has greater than 1, it shall be a multi-dimensional case.
+    """
+
+    REWARD_NAME = "name"
+    """Key for each dimension in multi-dimensional reward spans."""
+
+    REWARD_VALUE = "value"
+    """Value for each dimension in multi-dimensional reward spans."""
+
+
+class RewardPydanticModel(BaseModel):
+    """A stricter implementation of RewardAttributes used in otel helpers."""
+
+    name: str
+    """Name of the reward dimension."""
+
+    value: float
+    """Value of the reward dimension."""
+
+
+class LinkAttributes(Enum):
+    """Standard link types used in Agent-lightning spans.
+
+    The link is more powerful than [OpenTelemetry link](https://opentelemetry.io/docs/specs/otel/trace/api/#link)
+    in that it supports linking to a queryset of spans.
+    It can even link to span object that hasn't been emitted yet.
+    """
+
+    KEY_MATCH = "key_match"
+    """Linking to spans with matching attribute keys.
+
+    `trace_id` and `span_id` are reserved and will be used to link to specific spans directly.
+
+    For example, it can be `gen_ai.response.id` if intended to be link to a chat completion response span.
+    Or it can be `span_id` to link to a specific span by its ID.
+    """
+
+    VALUE_MATCH = "value_match"
+    """Linking to spans with corresponding attribute values on those keys."""
+
+
+class LinkPydanticModel(BaseModel):
+    """A stricter implementation of LinkAttributes used in otel helpers."""
+
+    key_match: str
+    """The attribute key to match on the target spans."""
+
+    value_match: str
+    """The attribute value to match on the target spans."""

agentlightning/store/__init__.py

@@ -1,15 +1,18 @@
 # Copyright (c) Microsoft. All rights reserved.
 
-from .base import LightningStore, LightningStoreCapabilities
+from .base import LightningStore, LightningStoreCapabilities, LightningStoreStatistics
 from .client_server import LightningStoreClient, LightningStoreServer
+from .collection_based import CollectionBasedLightningStore
 from .memory import InMemoryLightningStore
 from .threading import LightningStoreThreaded
 
 __all__ = [
     "LightningStore",
     "LightningStoreCapabilities",
+    "LightningStoreStatistics",
     "LightningStoreClient",
     "LightningStoreServer",
     "InMemoryLightningStore",
+    "CollectionBasedLightningStore",
     "LightningStoreThreaded",
 ]

agentlightning/store/base.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Any, Dict, List, Literal, Optional, Sequence, TypedDict
+from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, TypedDict
 
 from opentelemetry.sdk.trace import ReadableSpan
 
@@ -10,13 +10,17 @@ from agentlightning.types import (
     Attempt,
     AttemptedRollout,
     AttemptStatus,
+    EnqueueRolloutRequest,
     NamedResources,
     ResourcesUpdate,
     Rollout,
     RolloutConfig,
+    RolloutMode,
     RolloutStatus,
     Span,
     TaskInput,
+    Worker,
+    WorkerStatus,
 )
 
 
@@ -52,8 +56,11 @@ UNSET = _UnsetType()
 Unset = _UnsetType  # Alias for convenience
 
 
-class LightningStoreCapabilities(TypedDict):
-    """Capability of a LightningStore implementation."""
+class LightningStoreCapabilities(TypedDict, total=False):
+    """Capability of a LightningStore implementation.
+
+    All keys are optional and false by default.
+    """
 
     thread_safe: bool
     """Whether the store is thread-safe."""
@@ -61,6 +68,37 @@ class LightningStoreCapabilities(TypedDict):
     """Whether the store is async-safe."""
     zero_copy: bool
     """Whether the store has only one copy across all threads/processes."""
+    otlp_traces: bool
+    """Whether the store supports OTLP/HTTP traces."""
+
+
+class LightningStoreStatistics(TypedDict, total=False):
+    """Statistics of a LightningStore implementation."""
+
+    name: str
+    """Name of the store implementation."""
+    total_rollouts: int
+    """Total number of rollouts in the store."""
+    total_attempts: int
+    """Total number of attempts in the store."""
+    total_spans: int
+    """Total number of spans in the store."""
+    total_resources: int
+    """Total number of resources in the store."""
+    total_workers: int
+    """Total number of workers in the store."""
+    uptime: float
+    """Uptime of since the store has been started."""
+
+    # Memory-related statistics
+    total_span_bytes: int
+    """Total number of bytes of spans in the store."""
+    eviction_threshold_bytes: int
+    """Eviction threshold for spans in bytes."""
+    safe_threshold_bytes: int
+    """Safe threshold for spans in bytes."""
+    memory_capacity_bytes: int
+    """Memory capacity of the store in bytes."""
 
 
 class LightningStore:
@@ -85,21 +123,46 @@ class LightningStore:
     Unless stated otherwise, missing identifiers should result in a `ValueError`.
     """
 
+    @property
     def capabilities(self) -> LightningStoreCapabilities:
         """Return the capabilities of the store."""
         return LightningStoreCapabilities(
             thread_safe=False,
             async_safe=False,
             zero_copy=False,
+            otlp_traces=False,
         )
 
+    async def statistics(self) -> LightningStoreStatistics:
+        """Return the statistics of the store."""
+        return {
+            "name": self.__class__.__name__,
+        }
+
+    def otlp_traces_endpoint(self) -> str:
+        """Return the OTLP/HTTP traces endpoint of the store.
+
+        The traces can have rollout ID and attempt ID (and optionally sequence ID)
+        saved in the "resource" of the spans.
+        The store, if it supports OTLP, should be able to receive the traces and save them
+        via [`add_span`][agentlightning.LightningStore.add_span] or
+        [`add_otel_span`][agentlightning.LightningStore.add_otel_span].
+
+        The endpoint should be compatible with [OTLP HTTP protocol](https://opentelemetry.io/docs/specs/otlp/).
+        It's not necessarily compatible with OTLP gRPC protocol.
+
+        The returned endpoint will usually ends with `/v1/traces`.
+        """
+        raise NotImplementedError()
+
     async def start_rollout(
         self,
         input: TaskInput,
-        mode: Literal["train", "val", "test"] | None = None,
+        mode: RolloutMode | None = None,
         resources_id: str | None = None,
         config: RolloutConfig | None = None,
         metadata: Dict[str, Any] | None = None,
+        worker_id: str | None = None,
     ) -> AttemptedRollout:
         """Register a rollout and immediately create its first attempt.
 
@@ -122,6 +185,7 @@ class LightningStore:
             resources_id: Concrete resource snapshot to execute against; defaults to the latest stored snapshot.
             config: Rollout retry/timeout policy. Should default to a fresh [`RolloutConfig`][agentlightning.RolloutConfig].
             metadata: Free-form metadata persisted verbatim with the rollout.
+            worker_id: Optional worker identifier to associate the new attempt with.
 
         Returns:
             The fully-populated [`AttemptedRollout`][agentlightning.AttemptedRollout] including
@@ -167,7 +231,23 @@ class LightningStore:
         """
         raise NotImplementedError()
 
-    async def dequeue_rollout(self) -> Optional[AttemptedRollout]:
+    async def enqueue_many_rollouts(self, rollouts: Sequence[EnqueueRolloutRequest]) -> Sequence[Rollout]:
+        """Persist multiple rollouts in `queuing` state.
+
+        The implementation can delegate to [`enqueue_rollout()`][agentlightning.LightningStore.enqueue_rollout]
+        per request and preserves the input ordering. Subclasses can override to provide
+        more efficient bulk enqueue semantics.
+
+        Args:
+            rollouts: Rollout submission payloads mirroring [`enqueue_rollout()`][agentlightning.LightningStore.enqueue_rollout]'s
+                parameters. Each entry requires `input` and can optionally include other fields.
+
+        Returns:
+            Rollouts enqueued in the same order as `rollouts`.
+        """
+        raise NotImplementedError()
+
+    async def dequeue_rollout(self, worker_id: Optional[str] = None) -> Optional[AttemptedRollout]:
         """Claim the oldest queued rollout and transition it to `preparing`.
 
         This function do not block.
@@ -180,6 +260,11 @@ class LightningStore:
           the number of attempts already registered for the rollout plus one.
         * Return an [`AttemptedRollout`][agentlightning.AttemptedRollout] snapshot so the
           runner knows both rollout metadata and the attempt identifier.
+        * Optionally refresh the caller's [`Worker`][agentlightning.Worker] telemetry
+          (e.g., `last_dequeue_time`) when `worker_id` is provided.
+
+        Args:
+            worker_id: Optional worker identifier to associate the claimed attempt with.
 
         Returns:
             The next attempt to execute, or `None` when no eligible rollouts are queued.
@@ -189,7 +274,30 @@ class LightningStore:
         """
         raise NotImplementedError()
 
-    async def start_attempt(self, rollout_id: str) -> AttemptedRollout:
+    async def dequeue_many_rollouts(
+        self,
+        *,
+        limit: int = 1,
+        worker_id: Optional[str] = None,
+    ) -> Sequence[AttemptedRollout]:
+        """Claim up to `limit` queued rollouts without blocking.
+
+        The implementation can repeatedly invokes
+        [`dequeue_rollout()`][agentlightning.LightningStore.dequeue_rollout] until reaching
+        the requested limit or the queue is empty. Subclasses can override it to fetch
+        multiple rollouts atomically.
+
+        Args:
+            limit: Maximum number of rollouts to claim. Non-positive values return an empty list.
+            worker_id: Optional worker identifier passed through to each dequeue call.
+
+        Returns:
+            Attempted rollouts claimed in FIFO order. May contain fewer than `limit` entries
+            when the queue is exhausted.
+        """
+        raise NotImplementedError()
+
+    async def start_attempt(self, rollout_id: str, worker_id: Optional[str] = None) -> AttemptedRollout:
         """Create a manual retry attempt for an existing rollout.
 
         This is typically invoked by runners that wish to retry outside of the
@@ -200,6 +308,7 @@ class LightningStore:
 
         Args:
             rollout_id: Unique identifier of the rollout receiving a new attempt.
+            worker_id: Optional worker identifier to associate the new attempt with.
 
         Returns:
             The rollout paired with its newly-created attempt.
@@ -210,7 +319,15 @@ class LightningStore:
         """
         raise NotImplementedError()
 
-    async def add_span(self, span: Span) -> Span:
+    async def add_many_spans(self, spans: Sequence[Span]) -> Sequence[Span]:
+        """Persist a sequence of pre-constructed spans emitted during rollout execution.
+
+        Implementations can simply delegate to [`add_span()`][agentlightning.LightningStore.add_span] for each span.
+        However, if the store supports bulk insertion, it can implement this method to improve performance.
+        """
+        raise NotImplementedError()
+
+    async def add_span(self, span: Span) -> Optional[Span]:
         """Persist a pre-constructed span emitted during rollout execution.
 
         The provided [`Span`][agentlightning.Span] must already contain the `rollout_id`,
@@ -227,6 +344,7 @@ class LightningStore:
 
         Returns:
             The stored span record (implementations may return a copy).
+            Return `None` if the span was not added due to a duplicate.
 
         Raises:
             NotImplementedError: Subclasses must implement span persistence.
@@ -240,7 +358,7 @@ class LightningStore:
         attempt_id: str,
         readable_span: ReadableSpan,
         sequence_id: int | None = None,
-    ) -> Span:
+    ) -> Optional[Span]:
         """Convert and persist an OpenTelemetry span for a particular attempt.
 
         Implementations must transform the `readable_span` into a [`Span`][agentlightning.Span]
@@ -257,7 +375,7 @@ class LightningStore:
                 automatically.
 
         Returns:
-            The stored span record.
+            The stored span record. Return `None` if the span was not added due to a duplicate.
 
         Raises:
             NotImplementedError: Subclasses must implement span persistence.
@@ -266,30 +384,77 @@ class LightningStore:
         raise NotImplementedError()
 
     async def query_rollouts(
-        self, *, status: Optional[Sequence[RolloutStatus]] = None, rollout_ids: Optional[Sequence[str]] = None
-    ) -> List[Rollout]:
+        self,
+        *,
+        status_in: Optional[Sequence[RolloutStatus]] = None,
+        rollout_id_in: Optional[Sequence[str]] = None,
+        rollout_id_contains: Optional[str] = None,
+        filter_logic: Literal["and", "or"] = "and",
+        sort_by: Optional[str] = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
+        # Deprecated fields
+        status: Optional[Sequence[RolloutStatus]] = None,
+        rollout_ids: Optional[Sequence[str]] = None,
+    ) -> Sequence[Rollout]:
         """Retrieve rollouts filtered by status and/or explicit identifiers.
 
+        This interface supports structured filtering, sorting, and pagination so
+        callers can build simple dashboards without copying data out of the
+        store. The legacy parameters `status` and `rollout_ids` remain valid and
+        are treated as aliases for `status_in` and `rollout_id_in`
+        respectively—when both the new and deprecated parameters are supplied
+        the new parameters take precedence.
+
         Args:
-            status: Optional whitelist of [`RolloutStatus`][agentlightning.RolloutStatus] values.
-            rollout_ids: Optional whitelist of rollout identifiers to include.
+            status_in: Optional whitelist of [`RolloutStatus`][agentlightning.RolloutStatus] values.
+            rollout_id_in: Optional whitelist of rollout identifiers to include.
+            rollout_id_contains: Optional substring match for rollout identifiers.
+            filter_logic: Logical operator to combine filters.
+            sort_by: Optional field to sort by. Must reference a numeric or string
+                field on [`Rollout`][agentlightning.Rollout].
+            sort_order: Direction to sort when `sort_by` is provided.
+            limit: Maximum number of rows to return. Use `-1` for "no limit".
+            offset: Number of rows to skip before returning results.
+            status: Deprecated field. Use `status_in` instead.
+            rollout_ids: Deprecated field. Use `rollout_id_in` instead.
 
         Returns:
-            A list of matching rollouts. Ordering is backend-defined but must be deterministic.
+            A sequence of matching rollouts (or [`AttemptedRollout`][agentlightning.AttemptedRollout]
+            when attempts exist). Ordering is deterministic when `sort_by` is set.
+            The return value is not guaranteed to be a list.
 
         Raises:
             NotImplementedError: Subclasses must implement the query.
         """
         raise NotImplementedError()
 
-    async def query_attempts(self, rollout_id: str) -> List[Attempt]:
+    async def query_attempts(
+        self,
+        rollout_id: str,
+        *,
+        sort_by: Optional[str] = "sequence_id",
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
+    ) -> Sequence[Attempt]:
         """Return every attempt ever created for `rollout_id` in ascending sequence order.
 
+        The parameters allow callers to re-order or paginate the attempts so that
+        large retry histories can be streamed lazily.
+
         Args:
             rollout_id: Identifier of the rollout being inspected.
+            sort_by: Field to sort by. Must be a numeric or string field of
+                [`Attempt`][agentlightning.Attempt]. Defaults to `sequence_id` (oldest first).
+            sort_order: Order to sort by.
+            limit: Limit on the number of results. `-1` for unlimited.
+            offset: Offset into the results.
 
         Returns:
-            Attempts sorted by `sequence_id` (oldest first). Returns an empty list when none exist.
+            Sequence of Attempts. Returns an empty sequence when none exist.
+            The return value is not guaranteed to be a list.
 
         Raises:
             NotImplementedError: Subclasses must implement the query.
@@ -326,11 +491,35 @@ class LightningStore:
         """
         raise NotImplementedError()
 
-    async def query_resources(self) -> List[ResourcesUpdate]:
+    async def query_resources(
+        self,
+        *,
+        resources_id: Optional[str] = None,
+        resources_id_contains: Optional[str] = None,
+        # Filter logic is not supported here because I can't see why it's needed.
+        sort_by: Optional[str] = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
+    ) -> Sequence[ResourcesUpdate]:
         """List every stored resource snapshot in insertion order.
 
+        Supports lightweight filtering, sorting, and pagination for embedding in
+        dashboards.
+
+        Args:
+            resources_id: Optional identifier of the resources to include.
+            resources_id_contains: Optional substring match for resources identifiers.
+            sort_by: Optional field to sort by (must be numeric or string on
+                [`ResourcesUpdate`][agentlightning.ResourcesUpdate]).
+            sort_order: Order to sort by.
+            limit: Limit on the number of results. `-1` for unlimited.
+            offset: Offset into the results.
+
         Returns:
-            A chronological list of [`ResourcesUpdate`][agentlightning.ResourcesUpdate] objects.
+            [`ResourcesUpdate`][agentlightning.ResourcesUpdate] objects.
+            By default, resources are sorted in a deterministic but undefined order.
+            The return value is not guaranteed to be a list.
 
         Raises:
             NotImplementedError: Subclasses must implement retrieval.
@@ -387,6 +576,20 @@ class LightningStore:
         """
         raise NotImplementedError()
 
+    async def get_many_span_sequence_ids(self, rollout_attempt_ids: Sequence[Tuple[str, str]]) -> Sequence[int]:
+        """Bulk allocate the next strictly increasing sequence number used to order spans.
+
+        Implementations may delegate to [`get_next_span_sequence_id()`][agentlightning.LightningStore.get_next_span_sequence_id]
+        for each rollout and attempt.
+
+        Args:
+            rollout_attempt_ids: List of tuples of rollout and attempt identifiers.
+
+        Returns:
+            List of sequence numbers.
+        """
+        raise NotImplementedError()
+
     async def wait_for_rollouts(self, *, rollout_ids: List[str], timeout: Optional[float] = None) -> List[Rollout]:
         """Block until the targeted rollouts reach a terminal status or the timeout expires.
 
@@ -413,19 +616,61 @@ class LightningStore:
         """
         raise NotImplementedError()
 
-    async def query_spans(self, rollout_id: str, attempt_id: str | Literal["latest"] | None = None) -> List[Span]:
+    async def query_spans(
+        self,
+        rollout_id: str,
+        attempt_id: str | Literal["latest"] | None = None,
+        *,
+        # Filtering
+        trace_id: Optional[str] = None,
+        trace_id_contains: Optional[str] = None,
+        span_id: Optional[str] = None,
+        span_id_contains: Optional[str] = None,
+        parent_id: Optional[str] = None,
+        parent_id_contains: Optional[str] = None,
+        name: Optional[str] = None,
+        name_contains: Optional[str] = None,
+        filter_logic: Literal["and", "or"] = "and",
+        # Pagination
+        limit: int = -1,
+        offset: int = 0,
+        # Sorting
+        sort_by: Optional[str] = "sequence_id",
+        sort_order: Literal["asc", "desc"] = "asc",
+    ) -> Sequence[Span]:
         """Return the stored spans for a rollout, optionally scoped to one attempt.
 
-        Spans must be returned in ascending `sequence_id` order. Implementations may raise
-        a `RuntimeError` when spans were evicted or expired.
+        Supports a handful of filters that cover the most common debugging
+        scenarios (matching `trace_id`/`span_id`/`parent_id` or substring
+        matches on the span name). `attempt_id="latest"` acts as a convenience
+        that resolves the most recent attempt before evaluating filters. When
+        `attempt_id=None`, spans across every attempt are eligible. By default
+        results are sorted by `sequence_id` (oldest first). Implementations may
+        raise a `RuntimeError` when spans were evicted or expired.
 
         Args:
             rollout_id: Identifier of the rollout being inspected.
             attempt_id: Attempt identifier to filter by. Pass `"latest"` to retrieve only the
                 most recent attempt, or `None` to return all spans across attempts.
+            trace_id: Optional trace ID to filter by.
+            trace_id_contains: Optional substring match for trace IDs.
+            span_id: Optional span ID to filter by.
+            span_id_contains: Optional substring match for span IDs.
+            parent_id: Optional parent span ID to filter by.
+            parent_id_contains: Optional substring match for parent span IDs.
+            name: Optional span name to filter by.
+            name_contains: Optional substring match for span names.
+            filter_logic: Logical operator to combine the optional filters above.
+                The `rollout_id` argument is always applied with AND semantics.
+            limit: Limit on the number of results. `-1` for unlimited.
+            offset: Offset into the results.
+            sort_by: Field to sort by. Must be a numeric or string field of
+                [`Span`][agentlightning.Span].
+            sort_order: Order to sort by.
 
         Returns:
             An ordered list of spans (possibly empty).
+            The return value is not guaranteed to be a list.
 
         Raises:
             NotImplementedError: Subclasses must implement the query.
@@ -521,12 +766,19 @@ class LightningStore:
 
         When `attempt_id` is `"latest"` the update must target the attempt with the highest
         `sequence_id`; otherwise it must target the specific attempt. Implementations should
-        propagate status changes to the rollout (for example via [`propagate_status()`][agentlightning.store.utils.propagate_status])
+        propagate status changes to the rollout (for example
+        via [`rollout_status_from_attempt()`][agentlightning.store.utils.rollout_status_from_attempt])
         once the latest attempt transitions to a terminal state.
 
         Similar to [`update_rollout()`][agentlightning.LightningStore.update_rollout],
         parameters also default to the sentinel [`UNSET`][agentlightning.store.base.UNSET].
 
+        If `worker_id` is present, the worker status will be updated following the rules:
+
+        1. If attempt status is "succeeded" or "failed", the corresponding worker status will be set to "idle".
+        2. If attempt status is "unresponsive" or "timeout", the corresponding worker status will be set to "unknown".
+        3. Otherwise, the worker status will be set to "busy".
+
         Args:
             rollout_id: Identifier of the rollout whose attempt will be updated.
             attempt_id: Attempt identifier or `"latest"` as a convenience.
@@ -543,3 +795,64 @@ class LightningStore:
             ValueError: Implementations must raise when the rollout or attempt is unknown.
         """
         raise NotImplementedError()
+
+    async def query_workers(
+        self,
+        *,
+        status_in: Optional[Sequence[WorkerStatus]] = None,
+        worker_id_contains: Optional[str] = None,
+        filter_logic: Literal["and", "or"] = "and",
+        sort_by: Optional[str] = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
+    ) -> Sequence[Worker]:
+        """Query all workers in the system.
+
+        Args:
+            status_in: Optional whitelist of [`WorkerStatus`][agentlightning.WorkerStatus] values.
+            worker_id_contains: Optional substring match for worker identifiers.
+            filter_logic: Logical operator to combine the optional filters above.
+            sort_by: Field to sort by. Must be a numeric or string field of [`Worker`][agentlightning.Worker].
+            sort_order: Order to sort by.
+            limit: Limit on the number of results. `-1` for unlimited.
+            offset: Offset into the results.
+
+        Returns:
+            Sequence of Workers. Returns an empty sequence when none exist.
+            The return value is not guaranteed to be a list.
+        """
+        raise NotImplementedError()
+
+    async def get_worker_by_id(self, worker_id: str) -> Optional[Worker]:
+        """Retrieve a single worker by identifier.
+
+        Args:
+            worker_id: Identifier of the worker.
+
+        Returns:
+            The worker record if it exists, otherwise `None`.
+
+        Raises:
+            NotImplementedError: Subclasses must implement lookup semantics.
+        """
+        raise NotImplementedError()
+
+    async def update_worker(
+        self,
+        worker_id: str,
+        heartbeat_stats: Dict[str, Any] | Unset = UNSET,
+    ) -> Worker:
+        """Record a heartbeat for `worker_id` and refresh telemetry.
+
+        Implementations must treat this API as heartbeat-only: it should snapshot
+        the latest stats when provided, stamp `last_heartbeat_time` with the
+        current wall clock, and rely on other store mutations (`dequeue_rollout`,
+        `update_attempt`, etc.) to drive the worker's busy/idle status,
+        assignment, and activity timestamps.
+
+        Args:
+            worker_id: Identifier of the worker to update.
+            heartbeat_stats: Replacement worker heartbeat statistics (non-null when provided).
+        """
+        raise NotImplementedError()

agentlightning/store/collection/__init__.py

@@ -0,0 +1,30 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from .base import (
+    AtomicLabels,
+    AtomicMode,
+    Collection,
+    FilterOptions,
+    KeyValue,
+    LightningCollections,
+    PaginatedResult,
+    Queue,
+    SortOptions,
+)
+from .memory import DequeBasedQueue, DictBasedKeyValue, InMemoryLightningCollections, ListBasedCollection
+
+__all__ = [
+    "AtomicLabels",
+    "AtomicMode",
+    "Collection",
+    "Queue",
+    "KeyValue",
+    "FilterOptions",
+    "SortOptions",
+    "PaginatedResult",
+    "LightningCollections",
+    "ListBasedCollection",
+    "DequeBasedQueue",
+    "DictBasedKeyValue",
+    "InMemoryLightningCollections",
+]

agentlightning/store/collection/base.py

@@ -0,0 +1,414 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from __future__ import annotations
+
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncContextManager,
+    Awaitable,
+    Callable,
+    Dict,
+    Generic,
+    List,
+    Literal,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    TypeVar,
+    cast,
+)
+
+if TYPE_CHECKING:
+    from typing import Self
+
+from agentlightning.types import (
+    Attempt,
+    FilterField,
+    FilterOptions,
+    PaginatedResult,
+    ResourcesUpdate,
+    Rollout,
+    SortOptions,
+    Span,
+    Worker,
+)
+
+T = TypeVar("T")  # Recommended to be a BaseModel
+K = TypeVar("K")
+V = TypeVar("V")
+
+AtomicMode = Literal["r", "w", "rw"]
+"""What is expected within the atomic context. Can be "read", "write", or "read-write"."""
+
+AtomicLabels = Literal["rollouts", "attempts", "spans", "resources", "workers", "rollout_queue", "span_sequence_ids"]
+"""Labels for atomic operations.
+
+These labels are used to identify the collections that are affected by the atomic operation.
+"""
+
+
+class Collection(Generic[T]):
+    """Behaves like a list of items. Supporting addition, updating, and deletion of items."""
+
+    def primary_keys(self) -> Sequence[str]:
+        """Get the primary keys of the collection."""
+        raise NotImplementedError()
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}[{self.item_type().__name__}]>"
+
+    def item_type(self) -> Type[T]:
+        """Get the type of the items in the collection."""
+        raise NotImplementedError()
+
+    async def size(self) -> int:
+        """Get the number of items in the collection."""
+        raise NotImplementedError()
+
+    async def query(
+        self,
+        filter: Optional[FilterOptions] = None,
+        sort: Optional[SortOptions] = None,
+        limit: int = -1,
+        offset: int = 0,
+    ) -> PaginatedResult[T]:
+        """Query the collection with the given filters, sort order, and pagination.
+
+        Args:
+            filter:
+                The filters to apply to the collection. See [`FilterOptions`][agentlightning.FilterOptions].
+
+            sort:
+                The options for sorting the collection. See [`SortOptions`][agentlightning.SortOptions].
+                The field must exist in the model. If field might contain null values, in which case the behavior is undefined
+                (i.e., depending on the implementation).
+
+            limit:
+                Max number of items to return. Use -1 for "no limit".
+
+            offset:
+                Number of items to skip from the start of the *matching* items.
+
+        Returns:
+            PaginatedResult with items, limit, offset, and total matched items.
+        """
+        raise NotImplementedError()
+
+    async def get(
+        self,
+        filter: Optional[FilterOptions] = None,
+        sort: Optional[SortOptions] = None,
+    ) -> Optional[T]:
+        """Get the first item that matches the given filters.
+
+        Args:
+            filter: The filters to apply to the collection.
+                See [`FilterOptions`][agentlightning.store.collection.FilterOptions].
+            sort: Sort options. See [`SortOptions`][agentlightning.store.collection.SortOptions].
+
+        Returns:
+            The first item that matches the given filters, or None if no item matches.
+        """
+        raise NotImplementedError()
+
+    async def insert(self, items: Sequence[T]) -> None:
+        """Add the given items to the collection.
+
+        Raises:
+            ValueError: If an item with the same primary key already exists.
+        """
+        raise NotImplementedError()
+
+    async def update(self, items: Sequence[T], update_fields: Sequence[str] | None = None) -> Sequence[T]:
+        """Update the given items in the collection.
+
+        Args:
+            items: The items to update in the collection.
+            update_fields: The fields to update. If not provided, all fields in the type will be updated.
+                Only applicable if the item type is a Pydantic BaseModel.
+
+        Raises:
+            ValueError: If an item with the primary keys does not exist.
+
+        Returns:
+            The items that were updated.
+        """
+        raise NotImplementedError()
+
+    async def upsert(self, items: Sequence[T], update_fields: Sequence[str] | None = None) -> Sequence[T]:
+        """Upsert the given items into the collection.
+
+        If the items with the same primary keys already exist, they will be updated.
+        Otherwise, they will be inserted.
+
+        The operation has three semantics configurable via `update_fields`:
+
+        - `update_or_insert` via `collection.upsert(items, update_fields=["status", "updated_at"])`.
+          If the item with the same primary keys already exists, only the specified fields will be updated.
+          Otherwise, the item will be inserted.
+        - `get_or_insert` via `collection.upsert(items, update_fields=[])`.
+          If the item with the same primary keys already exists, the item will be left unchanged.
+          Otherwise, the item will be inserted.
+        - `replace_ish` via `collection.upsert(items)`.
+          If the item with the same primary keys already exists, all fields from the item will be set.
+          Otherwise, the item will be inserted.
+
+        Returns:
+            The items that were upserted.
+        """
+        raise NotImplementedError()
+
+    async def delete(self, items: Sequence[T]) -> None:
+        """Delete the given items from the collection.
+
+        Args:
+            items: The items to delete from the collection.
+
+        Raises:
+            ValueError: If the items with the primary keys to be deleted do not exist.
+        """
+        raise NotImplementedError()
+
+
+class Queue(Generic[T]):
+    """Behaves like a deque. Supporting appending items to the end and popping items from the front."""
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}[{self.item_type().__name__}]>"
+
+    def item_type(self) -> Type[T]:
+        """Get the type of the items in the queue."""
+        raise NotImplementedError()
+
+    async def has(self, item: T) -> bool:
+        """Check if the given item is in the queue."""
+        raise NotImplementedError()
+
+    async def enqueue(self, items: Sequence[T]) -> Sequence[T]:
+        """Append the given items to the end of the queue.
+
+        Args:
+            items: The items to append to the end of the queue.
+
+        Returns:
+            The items that were appended to the end of the queue.
+        """
+        raise NotImplementedError()
+
+    async def dequeue(self, limit: int = 1) -> Sequence[T]:
+        """Pop the given number of items from the front of the queue.
+
+        Args:
+            limit: The number of items to pop from the front of the queue.
+
+        Returns:
+            The items that were popped from the front of the queue.
+            If there are less than `limit` items in the queue, the remaining items will be returned.
+        """
+        raise NotImplementedError()
+
+    async def peek(self, limit: int = 1) -> Sequence[T]:
+        """Peek the given number of items from the front of the queue.
+
+        Args:
+            limit: The number of items to peek from the front of the queue.
+
+        Returns:
+            The items that were peeked from the front of the queue.
+            If there are less than `limit` items in the queue, the remaining items will be returned.
+        """
+        raise NotImplementedError()
+
+    async def size(self) -> int:
+        """Get the number of items in the queue."""
+        raise NotImplementedError()
+
+
+class KeyValue(Generic[K, V]):
+    """Behaves like a dictionary. Supporting addition, updating, and deletion of items."""
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}>"
+
+    async def has(self, key: K) -> bool:
+        """Check if the given key is in the dictionary."""
+        raise NotImplementedError()
+
+    async def get(self, key: K, default: V | None = None) -> V | None:
+        """Get the value for the given key, or the default value if the key is not found."""
+        raise NotImplementedError()
+
+    async def set(self, key: K, value: V) -> None:
+        """Set the value for the given key."""
+        raise NotImplementedError()
+
+    async def pop(self, key: K, default: V | None = None) -> V | None:
+        """Pop the value for the given key, or the default value if the key is not found."""
+        raise NotImplementedError()
+
+    async def size(self) -> int:
+        """Get the number of items in the dictionary."""
+        raise NotImplementedError()
+
+
+class LightningCollections:
+    """Collections of rollouts, attempts, spans, resources, and workers.
+
+    [LightningStore][agentlightning.LightningStore] implementations can use this as a storage base
+    to implement the store API.
+    """
+
+    @property
+    def rollouts(self) -> Collection[Rollout]:
+        """Collections of rollouts."""
+        raise NotImplementedError()
+
+    @property
+    def attempts(self) -> Collection[Attempt]:
+        """Collections of attempts."""
+        raise NotImplementedError()
+
+    @property
+    def spans(self) -> Collection[Span]:
+        """Collections of spans."""
+        raise NotImplementedError()
+
+    @property
+    def resources(self) -> Collection[ResourcesUpdate]:
+        """Collections of resources."""
+        raise NotImplementedError()
+
+    @property
+    def workers(self) -> Collection[Worker]:
+        """Collections of workers."""
+        raise NotImplementedError()
+
+    @property
+    def rollout_queue(self) -> Queue[str]:
+        """Queue of rollouts (tasks)."""
+        raise NotImplementedError()
+
+    @property
+    def span_sequence_ids(self) -> KeyValue[str, int]:
+        """Dictionary (counter) of span sequence IDs."""
+        raise NotImplementedError()
+
+    def atomic(
+        self,
+        *,
+        mode: AtomicMode = "rw",
+        snapshot: bool = False,
+        commit: bool = False,
+        labels: Optional[Sequence[AtomicLabels]] = None,
+        **kwargs: Any,
+    ) -> AsyncContextManager[Self]:
+        """Perform a atomic operation on the collections.
+
+        Subclass may use args and kwargs to support multiple levels of atomicity.
+        The arguments can be seen as tags. They only imply the behavior of the operation, not the implementation.
+
+        Args:
+            mode: The mode of atomicity. See [`AtomicMode`][agentlightning.store.collection.AtomicMode].
+            snapshot: Enable read snapshot for repeatable reads. Data consistency is guaranteed. The real behavior is implementation-dependent.
+            commit: Enable commitment for write operations. Unsuccessful operations will be rolled back depending on the implementation.
+                Recommend to use [`execute()`][agentlightning.store.collection.LightningCollections.execute] for this level to enable automatic retries.
+                Remember that the real behavior is implementation-dependent.
+            labels: Labels to add to the atomic operation (commonly used as lock names or collection names).
+            **kwargs: Keyword arguments to pass to the operation.
+        """
+        raise NotImplementedError()
+
+    async def execute(
+        self,
+        callback: Callable[[Self], Awaitable[T]],
+        *,
+        mode: AtomicMode = "rw",
+        snapshot: bool = False,
+        commit: bool = False,
+        labels: Optional[Sequence[AtomicLabels]] = None,
+        **kwargs: Any,
+    ) -> T:
+        """Execute the given callback within an atomic operation. Retry on transient errors is implied.
+
+        See [`atomic()`][agentlightning.store.collection.LightningCollections.atomic] for more details.
+        """
+        async with self.atomic(mode=mode, snapshot=snapshot, commit=commit, labels=labels, **kwargs) as collections:
+            return await callback(collections)
+
+
+FilterMap = Mapping[str, FilterField]
+
+
+def merge_must_filters(target: MutableMapping[str, FilterField], definition: Any) -> None:
+    """Normalize a `_must` filter group into the provided mapping.
+
+    Mainly for validation purposes.
+    """
+    if definition is None:
+        return
+
+    entries: List[Mapping[str, FilterField]] = []
+    if isinstance(definition, Mapping):
+        entries.append(cast(Mapping[str, FilterField], definition))
+    elif isinstance(definition, Sequence) and not isinstance(definition, (str, bytes)):
+        for entry in definition:  # type: ignore
+            if not isinstance(entry, Mapping):
+                raise TypeError("Each `_must` entry must be a mapping of field names to operators")
+            entries.append(cast(Mapping[str, FilterField], entry))
+    else:
+        raise TypeError("`_must` filters must be provided as a mapping or sequence of mappings")
+
+    for entry in entries:
+        for field_name, ops in entry.items():
+            existing = target.get(field_name, {})
+            merged_ops: Dict[str, Any] = dict(existing)
+            for op_name, expected in ops.items():
+                if op_name in merged_ops:
+                    raise ValueError(f"Duplicate operator '{op_name}' for field '{field_name}' in must filters")
+                merged_ops[op_name] = expected
+            target[field_name] = cast(FilterField, merged_ops)
+
+
+def normalize_filter_options(
+    filter_options: Optional[FilterOptions],
+) -> Tuple[Optional[FilterMap], Optional[FilterMap], Literal["and", "or"]]:
+    """Convert FilterOptions to the internal structure and resolve aggregate logic."""
+    if not filter_options:
+        return None, None, "and"
+
+    aggregate = cast(Literal["and", "or"], filter_options.get("_aggregate", "and"))
+    if aggregate not in ("and", "or"):
+        raise ValueError(f"Unsupported filter aggregate '{aggregate}'")
+
+    # Extract normalized filters and must filters from the filter options.
+    normalized: Dict[str, FilterField] = {}
+    must_filters: Dict[str, FilterField] = {}
+    for field_name, ops in filter_options.items():
+        if field_name == "_aggregate":
+            continue
+        if field_name == "_must":
+            merge_must_filters(must_filters, ops)
+            continue
+        normalized[field_name] = cast(FilterField, dict(ops))  # type: ignore
+
+    return (normalized or None, must_filters or None, aggregate)
+
+
+def resolve_sort_options(sort: Optional[SortOptions]) -> Tuple[Optional[str], Literal["asc", "desc"]]:
+    """Extract sort field/order from the caller-provided SortOptions."""
+    if not sort:
+        return None, "asc"
+
+    sort_name = sort.get("name")
+    if not sort_name:
+        raise ValueError("Sort options must include a 'name' field")
+
+    sort_order = sort.get("order", "asc")
+    if sort_order not in ("asc", "desc"):
+        raise ValueError(f"Unsupported sort order '{sort_order}'")
+
+    return sort_name, sort_order

agentlightning/store/collection/memory.py

@@ -0,0 +1,884 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+import weakref
+from collections import deque
+from contextlib import AsyncExitStack, asynccontextmanager
+from typing import (
+    Any,
+    Deque,
+    Dict,
+    Iterable,
+    List,
+    Literal,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
+
+import aiologic
+from pydantic import BaseModel
+
+from agentlightning.store.utils import LATENCY_BUCKETS
+from agentlightning.types import (
+    Attempt,
+    FilterField,
+    FilterOptions,
+    PaginatedResult,
+    ResourcesUpdate,
+    Rollout,
+    SortOptions,
+    Span,
+    Worker,
+)
+
+from .base import (
+    AtomicMode,
+    Collection,
+    FilterMap,
+    KeyValue,
+    LightningCollections,
+    Queue,
+    normalize_filter_options,
+    resolve_sort_options,
+)
+
+T = TypeVar("T")  # Recommended to be a BaseModel, not a dict
+K = TypeVar("K")
+V = TypeVar("V")
+
+logger = logging.getLogger(__name__)
+
+# Nested structure type:
+# dict[pk1] -> dict[pk2] -> ... -> item
+ListBasedCollectionItemType = Union[
+    Dict[Any, "ListBasedCollectionItemType[T]"],  # intermediate node
+    Dict[Any, T],  # leaf node dictionary
+]
+
+MutationMode = Literal["insert", "update", "upsert", "delete"]
+
+
+def _item_matches_filters(
+    item: object,
+    filters: Optional[FilterMap],
+    filter_logic: Literal["and", "or"],
+    must_filters: Optional[FilterMap] = None,
+) -> bool:
+    """Check whether an item matches the provided filter definition.
+
+    Filter format:
+
+    ```json
+    {
+        "_aggregate": "or",
+        "field_name": {
+            "exact": <value>,
+            "within": <iterable_of_allowed_values>,
+            "contains": <substring_or_element>,
+        },
+        ...
+    }
+    ```
+
+    Operators within the same field are stored in a unified pool and combined using
+    a universal logical operator.
+    """
+    if must_filters and not _item_matches_filters(item, must_filters, "and"):
+        return False
+
+    if not filters:
+        return True
+
+    all_conditions_match: List[bool] = []
+
+    for field_name, ops in filters.items():
+        item_value = getattr(item, field_name, None)
+
+        for op_name, expected in ops.items():
+            # Ignore no-op filters
+            if expected is None:
+                continue
+
+            if op_name == "exact":
+                all_conditions_match.append(item_value == expected)
+
+            elif op_name == "within":
+                try:
+                    all_conditions_match.append(item_value in expected)  # type: ignore[arg-type]
+                except TypeError:
+                    all_conditions_match.append(False)
+
+            elif op_name == "contains":
+                if item_value is None:
+                    all_conditions_match.append(False)
+                elif isinstance(item_value, str) and isinstance(expected, str):
+                    all_conditions_match.append(expected in item_value)
+                else:
+                    # Fallback: treat as generic iterable containment.
+                    try:
+                        all_conditions_match.append(expected in item_value)  # type: ignore[arg-type]
+                    except TypeError:
+                        all_conditions_match.append(False)
+            else:
+                raise ValueError(f"Unsupported filter operator '{op_name}' for field '{field_name}'")
+
+    return all(all_conditions_match) if filter_logic == "and" else any(all_conditions_match)
+
+
+def _get_sort_value(item: object, sort_by: str) -> Any:
+    """Get a sort key for the given item/field.
+
+    - If the field name ends with '_time', values are treated as comparable timestamps.
+    - For other fields we try to infer a safe default from the Pydantic model annotation.
+    """
+    value = getattr(item, sort_by, None)
+
+    if sort_by.endswith("_time"):
+        # For *_time fields, push missing values to the end.
+        return float("inf") if value is None else value
+
+    if value is None:
+        # Introspect model field type to choose a reasonable default for None.
+        model_fields = getattr(item.__class__, "model_fields", {})
+        if sort_by not in model_fields:
+            raise ValueError(
+                f"Failed to sort items by '{sort_by}': field does not exist " f"on {item.__class__.__name__}"
+            )
+
+        field_type_str = str(model_fields[sort_by].annotation)
+        if "str" in field_type_str or "Literal" in field_type_str:
+            return ""
+        if "int" in field_type_str:
+            return 0
+        if "float" in field_type_str:
+            return 0.0
+        raise ValueError(f"Failed to sort items by '{sort_by}': unsupported field type {field_type_str!r}")
+
+    return value
+
+
+class ListBasedCollection(Collection[T]):
+    """In-memory implementation of Collection using a nested dict for O(1) primary-key lookup.
+
+    The internal structure is:
+
+        {
+            pk1_value: {
+                pk2_value: {
+                    ...
+                        pkN_value: item
+                }
+            }
+        }
+
+    where the nesting depth equals the number of primary keys.
+
+    Sorting behavior:
+
+    1. If no sort_by is provided, the items are returned in the order of insertion.
+    2. If sort_by is provided, the items are sorted by the value of the sort_by field.
+    3. If the sort_by field is a timestamp, the null values are treated as infinity.
+    4. If the sort_by field is not a timestamp, the null values are treated as empty string
+       if the field is str-like, 0 if the field is int-like, 0.0 if the field is float-like.
+    """
+
+    def __init__(self, items: List[T], item_type: Type[T], primary_keys: Sequence[str]):
+        if not primary_keys:
+            raise ValueError("primary_keys must be non-empty")
+
+        self._items: Dict[Any, Any] = {}
+        self._size: int = 0
+        if issubclass(item_type, dict):
+            raise TypeError(f"Expect item to be not a dict, got {item_type.__name__}")
+        self._item_type: Type[T] = item_type
+        self._primary_keys: Tuple[str, ...] = tuple(primary_keys)
+
+        # Pre-populate the collection with the given items.
+        for item in items or []:
+            self._mutate_single(item, mode="insert")
+
+    def primary_keys(self) -> Sequence[str]:
+        """Return the primary key field names for this collection."""
+        return self._primary_keys
+
+    def item_type(self) -> Type[T]:
+        """Return the Pydantic model type of items stored in this collection."""
+        return self._item_type
+
+    async def size(self) -> int:
+        """Return the number of items stored in the collection."""
+        return self._size
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}[{self.item_type().__name__}] ({self._size})>"
+
+    # -------------------------------------------------------------------------
+    # Internal helpers
+    # -------------------------------------------------------------------------
+
+    def _ensure_item_type(self, item: T) -> None:
+        """Validate that the item matches the declared item_type."""
+        if not isinstance(item, self._item_type):
+            raise TypeError(f"Expected item of type {self._item_type.__name__}, " f"got {type(item).__name__}")
+
+    def _extract_primary_key_values(self, item: T) -> Tuple[Any, ...]:
+        """Extract the primary key values from an item.
+
+        Raises:
+            ValueError: If any primary key is missing on the item.
+        """
+        values: List[Any] = []
+        for key in self._primary_keys:
+            if not hasattr(item, key):
+                raise ValueError(f"Item {item} does not have primary key field '{key}'")
+            values.append(getattr(item, key))
+        return tuple(values)
+
+    def _render_key_values(self, key_values: Sequence[Any]) -> str:
+        return ", ".join(f"{name}={value!r}" for name, value in zip(self._primary_keys, key_values))
+
+    def _locate_node(
+        self,
+        key_values: Sequence[Any],
+        create_missing: bool,
+    ) -> Tuple[MutableMapping[Any, Any], Any]:
+        """Locate the parent mapping and final key for an item path.
+
+        Args:
+            key_values: The sequence of primary key values.
+            create_missing: Whether to create intermediate dictionaries as needed.
+
+        Returns:
+            (parent_mapping, final_key)
+
+        Raises:
+            KeyError: If the path does not exist and create_missing is False.
+            ValueError: If the internal structure is corrupted (non-dict where dict is expected).
+        """
+        if not key_values:
+            raise ValueError("key_values must be non-empty")
+
+        current: MutableMapping[Any, Any] = self._items
+        for idx, value in enumerate(key_values):
+            is_last = idx == len(key_values) - 1
+            if is_last:
+                # At the final level, current[value] is the item (or will be).
+                return current, value  # type: ignore
+
+            # Intermediate level: current[value] must be a dict.
+            if value not in current:
+                if not create_missing:
+                    raise KeyError(f"Path does not exist for given primary keys: {self._render_key_values(key_values)}")
+                current[value] = {}
+            next_node = current[value]  # type: ignore
+            if not isinstance(next_node, dict):
+                raise ValueError(f"Internal structure corrupted: expected dict, got {type(next_node)!r}")  # type: ignore
+            current = next_node  # type: ignore
+
+        # We should always return inside the loop.
+        raise RuntimeError("Unreachable")
+
+    def _mutate_single(self, item: T, mode: MutationMode, update_fields: Sequence[str] | None = None) -> Optional[T]:
+        """Core mutation logic shared by insert, update, upsert, and delete."""
+        self._ensure_item_type(item)
+        key_values = self._extract_primary_key_values(item)
+
+        if mode in ("insert", "upsert"):
+            parent, final_key = self._locate_node(key_values, create_missing=True)
+            exists = final_key in parent
+
+            if mode == "insert":
+                if exists:
+                    raise ValueError(f"Item already exists with primary key(s): {self._render_key_values(key_values)}")
+                parent[final_key] = item
+                self._size += 1
+            else:  # upsert
+                if not exists:
+                    self._size += 1
+                    parent[final_key] = item
+
+                elif update_fields is None:
+                    # update_or_insert: update all fields
+                    parent[final_key] = item
+
+                else:
+                    if not issubclass(self._item_type, BaseModel):
+                        raise TypeError(
+                            f"When using update_fields, the item type must be a Pydantic BaseModel, got {self._item_type.__name__}"
+                        )
+
+                    # Try to fetch the existing item
+                    existing = parent[final_key]
+                    if not isinstance(existing, self._item_type):
+                        raise ValueError(
+                            f"Internal structure corrupted: expected {self._item_type.__name__}, got {type(existing)!r}"
+                        )
+
+                    if not isinstance(item, self._item_type):
+                        raise TypeError(
+                            f"When using update_fields, the item type must be a Pydantic BaseModel, got {type(item).__name__}"
+                        )
+
+                    parent[final_key] = parent[final_key].model_copy(
+                        update={field: getattr(item, field) for field in update_fields}
+                    )
+
+            return parent[final_key]
+
+        elif mode in ("update", "delete"):
+            # For update/delete we must not create missing paths.
+            try:
+                parent, final_key = self._locate_node(key_values, create_missing=False)
+            except KeyError:
+                raise ValueError(
+                    f"Item does not exist with primary key(s): {self._render_key_values(key_values)}"
+                ) from None
+
+            if final_key not in parent:
+                raise ValueError(f"Item does not exist with primary key(s): {self._render_key_values(key_values)}")
+
+            if mode == "update":
+                if update_fields is None:
+                    # replace the entire item
+                    parent[final_key] = item
+                else:
+                    if not issubclass(self._item_type, BaseModel):
+                        raise TypeError(
+                            f"When using update_fields, the item type must be a Pydantic BaseModel, got {self._item_type.__name__}"
+                        )
+                    if not isinstance(item, self._item_type):
+                        raise TypeError(
+                            f"When using update_fields, the item type must be a Pydantic BaseModel, got {type(item).__name__}"
+                        )
+                    parent[final_key] = parent[final_key].model_copy(
+                        update={field: getattr(item, field) for field in update_fields}
+                    )
+                return parent[final_key]
+            else:  # delete
+                del parent[final_key]
+                self._size -= 1
+        else:
+            raise ValueError(f"Unknown mutation mode: {mode}")
+
+    def _iter_items(
+        self,
+        root: Optional[Mapping[Any, Any]] = None,
+        filters: Optional[FilterMap] = None,
+        must_filters: Optional[FilterMap] = None,
+        filter_logic: Literal["and", "or"] = "and",
+    ) -> Iterable[T]:
+        """Iterate over all items in the nested dictionary structure, optionally applying filters."""
+        if root is None:
+            root = self._items
+        if not root:
+            return
+        stack: List[Mapping[Any, Any]] = [root]
+        while stack:
+            node = stack.pop()
+            for value in node.values():
+                # Leaf nodes contain items; intermediate nodes are dicts.
+                if isinstance(value, self._item_type):
+                    if _item_matches_filters(value, filters, filter_logic, must_filters):
+                        yield value
+                elif isinstance(value, dict):
+                    stack.append(value)  # type: ignore
+                else:
+                    raise ValueError(
+                        f"Internal structure corrupted: expected dict or {self._item_type.__name__}, "
+                        f"got {type(value)!r}"
+                    )
+
+    def _iter_matching_items(
+        self,
+        filters: Optional[FilterMap],
+        must_filters: Optional[FilterMap],
+        filter_logic: Literal["and", "or"],
+    ) -> Iterable[T]:
+        """Efficiently iterate over items matching filters, using primary-key prefix when possible."""
+        # Fast path: when optional filters can't form a prefix, fall back to scanning.
+        if filter_logic != "and" and must_filters is None:
+            return self._iter_items(filters=filters, must_filters=must_filters, filter_logic=filter_logic)
+
+        # Try to derive a primary-key prefix from exact filters.
+        pk_values_prefix: List[Any] = []
+        prefix_sources: List[FilterMap] = []
+        if must_filters:
+            prefix_sources.append(must_filters)
+        if filter_logic == "and" and filters:
+            prefix_sources.append(filters)
+
+        for pk in self._primary_keys:
+            # combined_ops are: [{"exact": value}, {"within": [...]}, ...]
+            combined_ops: List[FilterField] = []
+            for source in prefix_sources:
+                field_ops = source.get(pk)  # type: ignore[union-attr]
+                if field_ops:
+                    combined_ops.append(field_ops)
+            if not combined_ops:
+                break
+            # Only allow a pure {"exact": value} constraint.
+            exact_value: Any | None = None
+            allow_prefix = True
+            for ops in combined_ops:
+                if set(ops.keys()) != {"exact"}:
+                    allow_prefix = False
+                    break
+                candidate = ops.get("exact")
+                if candidate is None:
+                    allow_prefix = False
+                    break
+                if exact_value is not None and candidate != exact_value:
+                    # Contradictory exact filters mean no items can match.
+                    logger.warning(f"Contradictory exact filters for field '{pk}': {exact_value} != {candidate}")
+                    return ()
+                exact_value = candidate
+
+            if not allow_prefix:
+                break
+
+            value = exact_value
+            if value is None:
+                break
+            pk_values_prefix.append(value)
+
+        if not pk_values_prefix:
+            return self._iter_items(filters=filters, must_filters=must_filters, filter_logic=filter_logic)
+
+        try:
+            if len(pk_values_prefix) == len(self._primary_keys):
+                # All primary keys specified -> at most a single item.
+                parent, final_key = self._locate_node(pk_values_prefix, create_missing=False)
+                single_item = parent.get(final_key)
+                if isinstance(single_item, self._item_type) and _item_matches_filters(
+                    single_item,
+                    filters,
+                    filter_logic,
+                    must_filters,
+                ):
+                    return (single_item,)
+                return ()
+            else:
+                # Prefix of primary keys specified -> iterate only the subtree below that prefix.
+                parent, final_key = self._locate_node(pk_values_prefix, create_missing=False)
+                subtree = parent.get(final_key)
+                if isinstance(subtree, dict):
+                    return self._iter_items(
+                        subtree,  # type: ignore
+                        filters=filters,
+                        must_filters=must_filters,
+                        filter_logic=filter_logic,
+                    )
+                return ()
+        except KeyError:
+            # No items exist for this primary-key prefix.
+            return ()
+
+    async def query(
+        self,
+        filter: Optional[FilterOptions] = None,
+        sort: Optional[SortOptions] = None,
+        limit: int = -1,
+        offset: int = 0,
+    ) -> PaginatedResult[T]:
+        """Query the collection with filters, sort order, and pagination.
+
+        Args:
+            filter: Mapping of field name to operator dict along with the optional `_aggregate` logic.
+            sort: Options describing which field to sort by and in which order.
+            limit: Max number of items to return. Use -1 for "no limit".
+            offset: Number of items to skip from the start of the *matching* items.
+        """
+        filters, must_filters, filter_logic = normalize_filter_options(filter)
+        sort_by, sort_order = resolve_sort_options(sort)
+        items_iter: Iterable[T] = self._iter_matching_items(filters, must_filters, filter_logic)
+
+        # No sorting: stream through items and apply pagination on the fly.
+        if not sort_by:
+            matched_items: List[T] = []
+            total_matched = 0
+
+            for item in items_iter:
+                # Count every match for 'total'
+                total_matched += 1
+
+                # Apply offset/limit window
+                if total_matched <= offset:
+                    continue
+                if limit != -1 and len(matched_items) >= limit:
+                    # Still need to finish iteration to get accurate total_matched.
+                    continue
+
+                matched_items.append(item)
+
+            return PaginatedResult(
+                items=matched_items,
+                limit=limit,
+                offset=offset,
+                total=total_matched,
+            )
+
+        # With sorting: we must materialize all matching items to sort them.
+        all_matches: List[T] = list(items_iter)
+
+        total_matched = len(all_matches)
+        reverse = sort_order == "desc"
+        all_matches.sort(key=lambda x: _get_sort_value(x, sort_by), reverse=reverse)
+
+        if limit == -1:
+            paginated_items = all_matches[offset:]
+        else:
+            paginated_items = all_matches[offset : offset + limit]
+
+        return PaginatedResult(
+            items=paginated_items,
+            limit=limit,
+            offset=offset,
+            total=total_matched,
+        )
+
+    async def get(
+        self,
+        filter: Optional[FilterOptions] = None,
+        sort: Optional[SortOptions] = None,
+    ) -> Optional[T]:
+        """Return the first (or best-sorted) item that matches the given filters, or None."""
+        filters, must_filters, filter_logic = normalize_filter_options(filter)
+        sort_by, sort_order = resolve_sort_options(sort)
+        items_iter: Iterable[T] = self._iter_matching_items(filters, must_filters, filter_logic)
+
+        if not sort_by:
+            # Just return the first matching item, if any.
+            for item in items_iter:
+                return item
+            return None
+
+        # Single-pass min/max according to sort_order.
+        best_item: Optional[T] = None
+        best_key: Any = None
+
+        for item in items_iter:
+            key = _get_sort_value(item, sort_by)
+            if best_item is None:
+                best_item = item
+                best_key = key
+                continue
+
+            if sort_order == "asc":
+                if key < best_key:
+                    best_item, best_key = item, key
+            else:
+                if key > best_key:
+                    best_item, best_key = item, key
+
+        return best_item
+
+    async def insert(self, items: Sequence[T]) -> None:
+        """Insert the given items.
+
+        Raises:
+            ValueError: If any item with the same primary keys already exists.
+        """
+        seen_keys: set[Tuple[Any, ...]] = set()
+        prepared: List[T] = []
+        for item in items:
+            self._ensure_item_type(item)
+            key_values = self._extract_primary_key_values(item)
+            if key_values in seen_keys:
+                raise ValueError(
+                    f"Insert payload contains duplicate primary key(s): {self._render_key_values(key_values)}"
+                )
+            seen_keys.add(key_values)
+            prepared.append(item)
+
+        for item in prepared:
+            self._mutate_single(item, mode="insert")
+
+    async def update(self, items: Sequence[T], update_fields: Sequence[str] | None = None) -> Sequence[T]:
+        """Update the given items.
+
+        Raises:
+            ValueError: If any item with the given primary keys does not exist.
+        """
+        updated_items: List[T] = []
+        for item in items:
+            updated = self._mutate_single(item, mode="update", update_fields=update_fields)
+            if updated is None:
+                raise RuntimeError(f"_mutate_single returned None for item {item}. This should never happen.")
+            updated_items.append(updated)
+        return updated_items
+
+    async def upsert(self, items: Sequence[T], update_fields: Sequence[str] | None = None) -> Sequence[T]:
+        """Upsert the given items (insert if missing, otherwise update)."""
+        upserted_items: List[T] = []
+        for item in items:
+            upserted = self._mutate_single(item, mode="upsert", update_fields=update_fields)
+            if upserted is None:
+                raise RuntimeError(f"_mutate_single returned None for item {item}. This should never happen.")
+            upserted_items.append(upserted)
+        return upserted_items
+
+    async def delete(self, items: Sequence[T]) -> None:
+        """Delete the given items.
+
+        Raises:
+            ValueError: If any item with the given primary keys does not exist.
+        """
+        # We use a two-phase approach to avoid partial deletion if one fails:
+        # first compute key_values to validate, then perform deletions.
+        for item in items:
+            # _mutate_single will validate existence and update size.
+            self._mutate_single(item, mode="delete")
+
+
+class DequeBasedQueue(Queue[T]):
+    """Queue implementation backed by collections.deque.
+
+    Provides O(1) amortized enqueue (append) and dequeue (popleft).
+    """
+
+    def __init__(self, item_type: Type[T], items: Optional[Sequence[T]] = None):
+        self._items: Deque[T] = deque()
+        self._item_type: Type[T] = item_type
+        if items:
+            self._items.extend(items)
+
+    def item_type(self) -> Type[T]:
+        return self._item_type
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}[{self.item_type().__name__}] ({len(self._items)})>"
+
+    async def has(self, item: T) -> bool:
+        if not isinstance(item, self._item_type):
+            raise TypeError(f"Expected item of type {self._item_type.__name__}, got {type(item).__name__}")
+        return item in self._items
+
+    async def enqueue(self, items: Sequence[T]) -> Sequence[T]:
+        for item in items:
+            if not isinstance(item, self._item_type):
+                raise TypeError(f"Expected item of type {self._item_type.__name__}, got {type(item).__name__}")
+            self._items.append(item)
+        return items
+
+    async def dequeue(self, limit: int = 1) -> Sequence[T]:
+        if limit <= 0:
+            return []
+        out: List[T] = []
+        for _ in range(min(limit, len(self._items))):
+            out.append(self._items.popleft())
+        return out
+
+    async def peek(self, limit: int = 1) -> Sequence[T]:
+        if limit <= 0:
+            return []
+        result: List[T] = []
+        count = min(limit, len(self._items))
+        for idx, item in enumerate(self._items):
+            if idx >= count:
+                break
+            result.append(item)
+        return result
+
+    async def size(self) -> int:
+        return len(self._items)
+
+
+class DictBasedKeyValue(KeyValue[K, V]):
+    """KeyValue implementation backed by a plain dictionary."""
+
+    def __init__(self, data: Optional[Mapping[K, V]] = None):
+        self._values: Dict[K, V] = dict(data) if data else {}
+
+    async def has(self, key: K) -> bool:
+        return key in self._values
+
+    async def get(self, key: K, default: V | None = None) -> V | None:
+        return self._values.get(key, default)
+
+    async def set(self, key: K, value: V) -> None:
+        self._values[key] = value
+
+    async def pop(self, key: K, default: V | None = None) -> V | None:
+        return self._values.pop(key, default)
+
+    async def size(self) -> int:
+        return len(self._values)
+
+
+class InMemoryLightningCollections(LightningCollections):
+    """In-memory implementation of LightningCollections using Python data structures.
+
+    Serves as the storage base for [`InMemoryLightningStore`][agentlightning.InMemoryLightningStore].
+    """
+
+    def __init__(self, lock_type: Literal["thread", "asyncio"], prometheus: bool = False):
+        self._lock = {
+            "rollouts": _LoopAwareAsyncLock() if lock_type == "asyncio" else _ThreadSafeAsyncLock(),
+            "attempts": _LoopAwareAsyncLock() if lock_type == "asyncio" else _ThreadSafeAsyncLock(),
+            "spans": _LoopAwareAsyncLock() if lock_type == "asyncio" else _ThreadSafeAsyncLock(),
+            "resources": _LoopAwareAsyncLock() if lock_type == "asyncio" else _ThreadSafeAsyncLock(),
+            "workers": _LoopAwareAsyncLock() if lock_type == "asyncio" else _ThreadSafeAsyncLock(),
+            "rollout_queue": _LoopAwareAsyncLock() if lock_type == "asyncio" else _ThreadSafeAsyncLock(),
+            "span_sequence_ids": _LoopAwareAsyncLock() if lock_type == "asyncio" else _ThreadSafeAsyncLock(),
+        }
+        self._rollouts = ListBasedCollection(items=[], item_type=Rollout, primary_keys=["rollout_id"])
+        self._attempts = ListBasedCollection(items=[], item_type=Attempt, primary_keys=["rollout_id", "attempt_id"])
+        self._spans = ListBasedCollection(
+            items=[], item_type=Span, primary_keys=["rollout_id", "attempt_id", "span_id"]
+        )
+        self._resources = ListBasedCollection(items=[], item_type=ResourcesUpdate, primary_keys=["resources_id"])
+        self._workers = ListBasedCollection(items=[], item_type=Worker, primary_keys=["worker_id"])
+        self._rollout_queue = DequeBasedQueue(items=[], item_type=str)
+        self._span_sequence_ids = DictBasedKeyValue[str, int](data={})  # rollout_id -> sequence_id
+
+        self._prometheus = prometheus
+        if self._prometheus:
+            from prometheus_client import Counter, Histogram
+
+            self._rate_metric = Counter(
+                "memory_collection_lock_rate",
+                "Rate of memory collection locks",
+                ["collection"],
+            )
+            self._latency_metric = Histogram(
+                "memory_collection_lock_latency_seconds",
+                "Latency of memory collection locks",
+                ["collection"],
+                buckets=LATENCY_BUCKETS,
+            )
+
+    @property
+    def rollouts(self) -> ListBasedCollection[Rollout]:
+        return self._rollouts
+
+    @property
+    def attempts(self) -> ListBasedCollection[Attempt]:
+        return self._attempts
+
+    @property
+    def spans(self) -> ListBasedCollection[Span]:
+        return self._spans
+
+    @property
+    def resources(self) -> ListBasedCollection[ResourcesUpdate]:
+        return self._resources
+
+    @property
+    def workers(self) -> ListBasedCollection[Worker]:
+        return self._workers
+
+    @property
+    def rollout_queue(self) -> DequeBasedQueue[str]:
+        return self._rollout_queue
+
+    @property
+    def span_sequence_ids(self) -> DictBasedKeyValue[str, int]:
+        return self._span_sequence_ids
+
+    @asynccontextmanager
+    async def atomic(
+        self, *, mode: AtomicMode = "rw", snapshot: bool = False, labels: Optional[Sequence[str]] = None, **kwargs: Any
+    ):
+        """In-memory collections apply a lock outside. It doesn't need to manipulate the collections inside.
+
+        Skip the locking if mode is "r" and snapshot is False.
+
+        This collection implementation does NOT support rollback / commit.
+        """
+        if mode == "r" and not snapshot:
+            yield self
+            return
+        if not labels:
+            # If no labels are provided, use all locks.
+            labels = list(self._lock.keys())
+
+        # IMPORTANT: Sort the labels to ensure consistent locking order.
+        # This is necessary to avoid deadlocks when multiple threads/coroutines
+        # are trying to acquire the same locks in different orders.
+        labels = sorted(labels)
+
+        managers = [(label, self._lock[label]) for label in labels]
+        async with AsyncExitStack() as stack:
+            for label, manager in managers:
+                start_time = time.perf_counter()
+                await stack.enter_async_context(manager)
+                elapsed = time.perf_counter() - start_time
+                if self._prometheus:
+                    self._rate_metric.labels(collection=label).inc()
+                    self._latency_metric.labels(collection=label).observe(elapsed)
+            yield self
+
+    async def evict_spans_for_rollout(self, rollout_id: str) -> None:
+        """Evict all spans for a given rollout ID.
+
+        Uses private API for efficiency.
+        """
+        self._spans._items.pop(rollout_id, [])  # pyright: ignore[reportPrivateUsage]
+
+
+class _LoopAwareAsyncLock:
+    """Async lock that transparently rebinds to the current event loop.
+
+    The lock intentionally remains *thread-unsafe*: callers must only use it from
+    one thread at a time. If multiple threads interact with the store, each
+    thread gets its own event loop specific lock.
+    """
+
+    def __init__(self) -> None:
+        self._locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock] = weakref.WeakKeyDictionary()
+
+    # When serializing and deserializing, we don't need to serialize the locks.
+    # Because another process will have its own set of event loops and its own lock.
+    def __getstate__(self) -> dict[str, Any]:
+        return {}
+
+    def __setstate__(self, state: dict[str, Any]) -> None:
+        self._locks = weakref.WeakKeyDictionary()
+
+    def _get_lock_for_current_loop(self) -> asyncio.Lock:
+        loop = asyncio.get_running_loop()
+        lock = self._locks.get(loop)
+        if lock is None:
+            lock = asyncio.Lock()
+            self._locks[loop] = lock
+        return lock
+
+    async def __aenter__(self) -> asyncio.Lock:
+        lock = self._get_lock_for_current_loop()
+        await lock.acquire()
+        return lock
+
+    async def __aexit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: Any) -> None:
+        loop = asyncio.get_running_loop()
+        lock = self._locks.get(loop)
+        if lock is None or not lock.locked():
+            raise RuntimeError("Lock released without being acquired")
+        lock.release()
+
+
+class _ThreadSafeAsyncLock:
+    """A thread lock powered by aiologic that can be used in both async and sync contexts.
+
+    aiologic claims itself to be a thread-safe asyncio lock.
+    """
+
+    def __init__(self):
+        self._lock = aiologic.Lock()
+
+    async def __aenter__(self):
+        await self._lock.async_acquire()
+        return self
+
+    async def __aexit__(self, *args: Any, **kwargs: Any):
+        # .release() is non-blocking, so we can call it directly
+        self._lock.async_release()

agentlightning/store/mongo.py

@@ -0,0 +1,165 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+import time
+import uuid
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    TypeVar,
+    Union,
+)
+
+from pymongo import AsyncMongoClient
+
+from agentlightning.types import Attempt, AttemptedRollout, Rollout
+
+from .base import LightningStoreCapabilities, is_finished
+from .collection.mongo import MongoClientPool, MongoLightningCollections, MongoOperationPrometheusTracker
+from .collection_based import CollectionBasedLightningStore, healthcheck_before, tracked
+
+T_callable = TypeVar("T_callable", bound=Callable[..., Any])
+
+logger = logging.getLogger(__name__)
+
+
+def _generate_partition_id() -> str:
+    return "pt-" + hashlib.sha1(uuid.uuid4().bytes).hexdigest()[:12]
+
+
+class MongoLightningStore(CollectionBasedLightningStore[MongoLightningCollections]):
+    """
+    MongoDB implementation of LightningStore using MongoDB collections.
+    Data is persistent and can be shared between multiple processes.
+
+    Args:
+        client: The MongoDB client. Could be a string URI or an instance of AsyncMongoClient.
+        database: The MongoDB database. Could be a string name or an instance of AsyncDatabase.
+            You must provide at least one of client or database.
+        partition_id: The partition id. Useful when sharing the database among multiple Agent-lightning trainers.
+    """
+
+    def __init__(
+        self,
+        *,
+        client: AsyncMongoClient[Mapping[str, Any]] | str,
+        database_name: str | None = None,
+        partition_id: str | None = None,
+        prometheus: bool = False,
+    ) -> None:
+        self._enable_prometheus = prometheus
+        self._auto_created_client = False
+        if isinstance(client, str):
+            self._client = AsyncMongoClient[Mapping[str, Any]](client)
+            self._auto_created_client = True
+        else:
+            self._client = client
+        if database_name is None:
+            database_name = "agentlightning"
+            logger.info("No database name provided, using default 'agentlightning'")
+
+        if partition_id is None:
+            partition_id = _generate_partition_id()
+            logger.info("No partition id provided, generated a new one: %s", partition_id)
+
+        self._client_pool = MongoClientPool(self._client)
+
+        super().__init__(
+            collections=MongoLightningCollections(
+                self._client_pool,
+                database_name,
+                partition_id,
+                prometheus_tracker=MongoOperationPrometheusTracker(enabled=self._enable_prometheus),
+            ),
+            prometheus=self._enable_prometheus,
+        )
+
+    @property
+    def capabilities(self) -> LightningStoreCapabilities:
+        """Return the capabilities of the store."""
+        return LightningStoreCapabilities(
+            thread_safe=True,
+            async_safe=True,
+            zero_copy=True,
+            otlp_traces=False,
+        )
+
+    async def close(self) -> None:
+        """Close the store by closing the client pool."""
+        await self._client_pool.close()
+        # If I created the client, I should close it too.
+        if self._auto_created_client:
+            await self._client.close()
+
+    @tracked("wait_for_rollouts")
+    @healthcheck_before
+    async def wait_for_rollouts(self, *, rollout_ids: List[str], timeout: Optional[float] = None) -> List[Rollout]:
+        """Wait for specified rollouts to complete with a timeout.
+
+        Concurrently wait for all rollouts to complete with a timeout.
+        """
+        start_time = time.time()
+        current_time = start_time
+        deadline = start_time + timeout if timeout is not None else None
+
+        finished_rollouts: Dict[str, Rollout] = {}
+        unfinished_rollout_ids = set(rollout_ids)
+
+        while deadline is None or current_time <= deadline:
+            async with self.collections.atomic(
+                mode="r", snapshot=self._read_snapshot, labels=["rollouts"]
+            ) as collections:
+                # Query the rollouts that are not finished in a single query
+                rollouts = await collections.rollouts.query(
+                    filter={"rollout_id": {"within": list(unfinished_rollout_ids)}}
+                )
+            for rollout in rollouts.items:
+                if is_finished(rollout):
+                    finished_rollouts[rollout.rollout_id] = rollout
+                    unfinished_rollout_ids.remove(rollout.rollout_id)
+
+            if not unfinished_rollout_ids:
+                break
+
+            # Poll every 10 seconds by default
+            # Minus 0.1 to make sure the time is still sufficient for another call
+            rest_time = max(0.01, min(deadline - time.time() - 0.1, 10.0)) if deadline is not None else 10.0
+            await asyncio.sleep(rest_time)
+            current_time = time.time()
+
+        # Reorder the rollouts to match the input order
+        return [finished_rollouts[rollout_id] for rollout_id in rollout_ids if rollout_id in finished_rollouts]
+
+    @tracked("_unlocked_many_rollouts_to_attempted_rollouts")
+    async def _unlocked_many_rollouts_to_attempted_rollouts(
+        self, collections: MongoLightningCollections, rollouts: Sequence[Rollout]
+    ) -> List[Union[Rollout, AttemptedRollout]]:
+        """Query the latest attempts for the rollouts, and attach them to the rollout objects."""
+        async with collections.atomic(mode="r", snapshot=self._read_snapshot, labels=["attempts"]) as collections:
+            attempts = await collections.attempts.query(
+                filter={"rollout_id": {"within": [rollout.rollout_id for rollout in rollouts]}},
+                sort={"name": "sequence_id", "order": "desc"},
+            )
+        latest_attempts: Dict[str, Attempt] = {}
+        for attempt in attempts:
+            if attempt.rollout_id not in latest_attempts:
+                latest_attempts[attempt.rollout_id] = attempt
+            # Otherwise we ignore the attempt because there's already a newer attempt
+
+        return [
+            (
+                AttemptedRollout(**rollout.model_dump(), attempt=latest_attempts[rollout.rollout_id])
+                if rollout.rollout_id in latest_attempts
+                else rollout
+            )
+            for rollout in rollouts
+        ]

agentlightning/store/threading.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import threading
-from typing import Any, Dict, List, Literal, Optional, Sequence
+from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple
 
 from opentelemetry.sdk.trace import ReadableSpan
 
@@ -11,6 +11,7 @@ from agentlightning.types import (
     Attempt,
     AttemptedRollout,
     AttemptStatus,
+    EnqueueRolloutRequest,
     NamedResources,
     ResourcesUpdate,
     Rollout,
@@ -18,9 +19,11 @@ from agentlightning.types import (
     RolloutStatus,
     Span,
     TaskInput,
+    Worker,
+    WorkerStatus,
 )
 
-from .base import UNSET, LightningStore, LightningStoreCapabilities, Unset
+from .base import UNSET, LightningStore, LightningStoreCapabilities, LightningStoreStatistics, Unset
 
 
 class LightningStoreThreaded(LightningStore):
@@ -35,15 +38,21 @@ class LightningStoreThreaded(LightningStore):
         self.store = store
         self._lock = threading.Lock()
 
+    @property
     def capabilities(self) -> LightningStoreCapabilities:
         """Return the capabilities of the store."""
-        capabilities = self.store.capabilities()
+        capabilities = self.store.capabilities
         return {
             **capabilities,
             "async_safe": True,
             "thread_safe": True,
         }
 
+    async def statistics(self) -> LightningStoreStatistics:
+        """Return the statistics of the store."""
+        with self._lock:
+            return await self.store.statistics()
+
     async def start_rollout(
         self,
         input: TaskInput,
@@ -51,9 +60,17 @@ class LightningStoreThreaded(LightningStore):
         resources_id: str | None = None,
         config: RolloutConfig | None = None,
         metadata: Dict[str, Any] | None = None,
+        worker_id: Optional[str] = None,
     ) -> AttemptedRollout:
         with self._lock:
-            return await self.store.start_rollout(input, mode, resources_id, config, metadata)
+            return await self.store.start_rollout(
+                input,
+                mode,
+                resources_id,
+                config,
+                metadata,
+                worker_id,
+            )
 
     async def enqueue_rollout(
         self,
@@ -66,26 +83,72 @@ class LightningStoreThreaded(LightningStore):
         with self._lock:
             return await self.store.enqueue_rollout(input, mode, resources_id, config, metadata)
 
-    async def dequeue_rollout(self) -> Optional[AttemptedRollout]:
+    async def enqueue_many_rollouts(self, rollouts: Sequence[EnqueueRolloutRequest]) -> Sequence[Rollout]:
         with self._lock:
-            return await self.store.dequeue_rollout()
+            return await self.store.enqueue_many_rollouts(rollouts)
 
-    async def start_attempt(self, rollout_id: str) -> AttemptedRollout:
+    async def dequeue_rollout(self, worker_id: Optional[str] = None) -> Optional[AttemptedRollout]:
         with self._lock:
-            return await self.store.start_attempt(rollout_id)
+            return await self.store.dequeue_rollout(worker_id=worker_id)
+
+    async def dequeue_many_rollouts(
+        self,
+        *,
+        limit: int = 1,
+        worker_id: Optional[str] = None,
+    ) -> Sequence[AttemptedRollout]:
+        with self._lock:
+            return await self.store.dequeue_many_rollouts(limit=limit, worker_id=worker_id)
+
+    async def start_attempt(self, rollout_id: str, worker_id: Optional[str] = None) -> AttemptedRollout:
+        with self._lock:
+            return await self.store.start_attempt(rollout_id, worker_id)
 
     async def query_rollouts(
         self,
         *,
+        status_in: Optional[Sequence[RolloutStatus]] = None,
+        rollout_id_in: Optional[Sequence[str]] = None,
+        rollout_id_contains: Optional[str] = None,
+        filter_logic: Literal["and", "or"] = "and",
+        sort_by: Optional[str] = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
         status: Optional[Sequence[RolloutStatus]] = None,
         rollout_ids: Optional[Sequence[str]] = None,
-    ) -> List[Rollout]:
+    ) -> Sequence[Rollout]:
         with self._lock:
-            return await self.store.query_rollouts(status=status, rollout_ids=rollout_ids)
+            return await self.store.query_rollouts(
+                status_in=status_in,
+                rollout_id_in=rollout_id_in,
+                rollout_id_contains=rollout_id_contains,
+                filter_logic=filter_logic,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                limit=limit,
+                offset=offset,
+                status=status,
+                rollout_ids=rollout_ids,
+            )
 
-    async def query_attempts(self, rollout_id: str) -> List[Attempt]:
+    async def query_attempts(
+        self,
+        rollout_id: str,
+        *,
+        sort_by: Optional[str] = "sequence_id",
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
+    ) -> Sequence[Attempt]:
         with self._lock:
-            return await self.store.query_attempts(rollout_id)
+            return await self.store.query_attempts(
+                rollout_id,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                limit=limit,
+                offset=offset,
+            )
 
     async def get_rollout_by_id(self, rollout_id: str) -> Optional[Rollout]:
         with self._lock:
@@ -95,6 +158,26 @@ class LightningStoreThreaded(LightningStore):
         with self._lock:
             return await self.store.get_latest_attempt(rollout_id)
 
+    async def query_resources(
+        self,
+        *,
+        resources_id: Optional[str] = None,
+        resources_id_contains: Optional[str] = None,
+        sort_by: Optional[str] = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
+    ) -> Sequence[ResourcesUpdate]:
+        with self._lock:
+            return await self.store.query_resources(
+                resources_id=resources_id,
+                resources_id_contains=resources_id_contains,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                limit=limit,
+                offset=offset,
+            )
+
     async def add_resources(self, resources: NamedResources) -> ResourcesUpdate:
         with self._lock:
             return await self.store.add_resources(resources)
@@ -111,7 +194,11 @@ class LightningStoreThreaded(LightningStore):
         with self._lock:
             return await self.store.get_latest_resources()
 
-    async def add_span(self, span: Span) -> Span:
+    async def add_many_spans(self, spans: Sequence[Span]) -> Sequence[Span]:
+        with self._lock:
+            return await self.store.add_many_spans(spans)
+
+    async def add_span(self, span: Span) -> Optional[Span]:
         with self._lock:
             return await self.store.add_span(span)
 
@@ -121,7 +208,7 @@ class LightningStoreThreaded(LightningStore):
         attempt_id: str,
         readable_span: ReadableSpan,
         sequence_id: int | None = None,
-    ) -> Span:
+    ) -> Optional[Span]:
         with self._lock:
             return await self.store.add_otel_span(rollout_id, attempt_id, readable_span, sequence_id)
 
@@ -133,13 +220,47 @@ class LightningStoreThreaded(LightningStore):
         with self._lock:
             return await self.store.get_next_span_sequence_id(rollout_id, attempt_id)
 
+    async def get_many_span_sequence_ids(self, rollout_attempt_ids: Sequence[Tuple[str, str]]) -> Sequence[int]:
+        with self._lock:
+            return await self.store.get_many_span_sequence_ids(rollout_attempt_ids)
+
     async def query_spans(
         self,
         rollout_id: str,
         attempt_id: str | Literal["latest"] | None = None,
-    ) -> List[Span]:
+        *,
+        trace_id: Optional[str] = None,
+        trace_id_contains: Optional[str] = None,
+        span_id: Optional[str] = None,
+        span_id_contains: Optional[str] = None,
+        parent_id: Optional[str] = None,
+        parent_id_contains: Optional[str] = None,
+        name: Optional[str] = None,
+        name_contains: Optional[str] = None,
+        filter_logic: Literal["and", "or"] = "and",
+        limit: int = -1,
+        offset: int = 0,
+        sort_by: Optional[str] = "sequence_id",
+        sort_order: Literal["asc", "desc"] = "asc",
+    ) -> Sequence[Span]:
         with self._lock:
-            return await self.store.query_spans(rollout_id, attempt_id)
+            return await self.store.query_spans(
+                rollout_id,
+                attempt_id,
+                trace_id=trace_id,
+                trace_id_contains=trace_id_contains,
+                span_id=span_id,
+                span_id_contains=span_id_contains,
+                parent_id=parent_id,
+                parent_id_contains=parent_id_contains,
+                name=name,
+                name_contains=name_contains,
+                filter_logic=filter_logic,
+                limit=limit,
+                offset=offset,
+                sort_by=sort_by,
+                sort_order=sort_order,
+            )
 
     async def update_rollout(
         self,
@@ -180,3 +301,39 @@ class LightningStoreThreaded(LightningStore):
                 last_heartbeat_time=last_heartbeat_time,
                 metadata=metadata,
             )
+
+    async def query_workers(
+        self,
+        *,
+        status_in: Optional[Sequence[WorkerStatus]] = None,
+        worker_id_contains: Optional[str] = None,
+        filter_logic: Literal["and", "or"] = "and",
+        sort_by: Optional[str] = None,
+        sort_order: Literal["asc", "desc"] = "asc",
+        limit: int = -1,
+        offset: int = 0,
+    ) -> Sequence[Worker]:
+        with self._lock:
+            return await self.store.query_workers(
+                status_in=status_in,
+                worker_id_contains=worker_id_contains,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                limit=limit,
+                offset=offset,
+            )
+
+    async def get_worker_by_id(self, worker_id: str) -> Optional[Worker]:
+        with self._lock:
+            return await self.store.get_worker_by_id(worker_id)
+
+    async def update_worker(
+        self,
+        worker_id: str,
+        heartbeat_stats: Dict[str, Any] | Unset = UNSET,
+    ) -> Worker:
+        with self._lock:
+            return await self.store.update_worker(
+                worker_id=worker_id,
+                heartbeat_stats=heartbeat_stats,
+            )

agentlightning/store/utils.py

@@ -1,7 +1,7 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 import time
-from typing import Awaitable, Callable, List, cast
+from typing import Awaitable, Callable, Dict, List, Tuple
 
 from agentlightning.types import Attempt, AttemptedRollout, AttemptStatus, Rollout, RolloutConfig, RolloutStatus
 
@@ -9,66 +9,102 @@ UpdateRolloutStatus = Callable[[str, RolloutStatus], Awaitable[Rollout]]
 UpdateAttemptStatus = Callable[[str, str, AttemptStatus], Awaitable[Attempt]]
 
 
-async def propagate_status(
-    update_rollout_status: UpdateRolloutStatus,  # this should be unlocked
+LATENCY_BUCKETS = [
+    0.000001,
+    0.000002,
+    0.000005,
+    0.00001,
+    0.00002,
+    0.00005,
+    0.0001,
+    0.0002,
+    0.0005,
+    0.001,
+    0.002,
+    0.003,
+    0.005,
+    0.007,
+    0.01,
+    0.015,
+    0.02,
+    0.03,
+    0.05,
+    0.07,
+    0.1,
+    0.2,
+    0.3,
+    0.5,
+    0.7,
+    1.0,
+    2.0,
+    3.0,
+    5.0,
+    7.0,
+    10.0,
+    12.0,
+    15.0,
+    20.0,
+    25.0,
+    30.0,
+    40.0,
+    50.0,
+    60.0,
+    90.0,
+    120.0,
+    180.0,
+    240.0,
+    300.0,
+]
+
+
+async def rollout_status_from_attempt(
     attempt: Attempt,
     config: RolloutConfig,
-) -> Rollout:
+) -> RolloutStatus:
     """
     Propagate the status of an attempt to the rollout.
 
-    The rollout should be made sure in a state to be outdated.
-    Requeue the rollout if it should be retried.
-
-    This operation is completely unlocked. The caller is responsible for locking the store.
+    Returns:
+        The status of the rollout from the perspective of the attempt.
     """
     # Propagate the status directly to the rollout
     if attempt.status == "preparing" or attempt.status == "running" or attempt.status == "succeeded":
-        return await update_rollout_status(
-            attempt.rollout_id,
-            attempt.status,
-        )
+        return attempt.status
 
     if attempt.status == "failed" or attempt.status == "timeout" or attempt.status == "unresponsive":
         # Check if this status should trigger a retry
         if attempt.status in config.retry_condition:
             # If we haven't exceeded max attempts, retry
             if attempt.sequence_id < config.max_attempts:
-                return await update_rollout_status(
-                    attempt.rollout_id,
-                    "requeuing",
-                )
+                return "requeuing"
 
         # If we can't retry or shouldn't retry, mark as failed
-        return await update_rollout_status(
-            attempt.rollout_id,
-            "failed",
-        )
+        return "failed"
 
     raise ValueError(f"Invalid attempt status: {attempt.status}")
 
 
-async def healthcheck(
+async def scan_unhealthy_rollouts(
     rollouts: List[AttemptedRollout],
-    update_rollout_status: UpdateRolloutStatus,
-    update_attempt_status: UpdateAttemptStatus,
-) -> None:
+) -> Dict[Tuple[str, str], AttemptStatus]:
     """
     Perform health check on all running rollouts in the store.
 
     This method should be called periodically to:
 
-    1. Update rollout status to failed to succeeded when the attempt is done
-    2. Check for unresponsive attempts (no heartbeat or spans for a while)
-    3. Check for timed-out rollouts (running too long since start_time)
-    4. Update attempt/rollout status accordingly
+    1. Check for unresponsive attempts (no heartbeat or spans for a while)
+    2. Check for timed-out rollouts (running too long since start_time)
 
     This operation is completely unlocked. The caller is responsible for locking the store.
 
     Args:
-        store: The LightningStore instance to check rollouts from
+        rollouts: The list of running rollouts to check.
+
+    Returns:
+        A dictionary of updates to the rollouts.
     """
     current_time = time.time()
+    updates: Dict[Tuple[str, str], AttemptStatus] = {}
 
     for rollout in rollouts:
         config = rollout.config  # policy for retry and timeout
@@ -76,52 +112,31 @@ async def healthcheck(
         # Get the latest attempt for this rollout
         latest_attempt = rollout.attempt
         if not latest_attempt:
-            continue
-
-        # Check if the attempt has already failed or succeeded
-        if latest_attempt.status == "failed" or latest_attempt.status == "succeeded":
-            await propagate_status(update_rollout_status, latest_attempt, config)
+            # This should not happen
             continue
 
         # Check for timeout condition (based on attempt start_time, instead of rollout start_time)
         if config.timeout_seconds is not None and current_time - latest_attempt.start_time > config.timeout_seconds:
-            await update_attempt_status(
-                latest_attempt.rollout_id,
-                latest_attempt.attempt_id,
-                "timeout",
-            )
+            updates[(latest_attempt.rollout_id, latest_attempt.attempt_id)] = "timeout"
             continue
 
         # Check for unresponsive condition (based on last heartbeat)
-        if latest_attempt.last_heartbeat_time:
-            if latest_attempt.status == "preparing":
-                # If still preparing, mark it as running
-                latest_attempt = await update_attempt_status(
-                    latest_attempt.rollout_id,
-                    latest_attempt.attempt_id,
-                    "running",
-                )
+        # (1) Haven't received heartbeat for a while
+        if (
+            latest_attempt.last_heartbeat_time
+            and config.unresponsive_seconds is not None
+            and current_time - latest_attempt.last_heartbeat_time > config.unresponsive_seconds
+        ):
+            updates[(latest_attempt.rollout_id, latest_attempt.attempt_id)] = "unresponsive"
+            continue
 
-            # Haven't received heartbeat for a while
-            if (
-                config.unresponsive_seconds is not None
-                and current_time - cast(float, latest_attempt.last_heartbeat_time) > config.unresponsive_seconds
-            ):
-                await update_attempt_status(
-                    latest_attempt.rollout_id,
-                    latest_attempt.attempt_id,
-                    "unresponsive",
-                )
-                continue
-
-        # Check if there's no last heartbeat (no spans) at all
+        # (2) Check if there's no last heartbeat (no spans) at all
         if (
             latest_attempt.last_heartbeat_time is None
             and config.unresponsive_seconds is not None
             and current_time - latest_attempt.start_time > config.unresponsive_seconds
         ):
-            await update_attempt_status(
-                latest_attempt.rollout_id,
-                latest_attempt.attempt_id,
-                "unresponsive",
-            )
+            updates[(latest_attempt.rollout_id, latest_attempt.attempt_id)] = "unresponsive"
+            continue
+
+    return updates

agentlightning/tracer/agentops.py

@@ -2,25 +2,24 @@
 
 from __future__ import annotations
 
-import asyncio
 import logging
 import os
-import threading
+import warnings
 from contextlib import asynccontextmanager, contextmanager
-from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Iterator, List, Optional
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Iterator, List, Optional
 
 import agentops
 import agentops.sdk.core
+import opentelemetry.trace as trace_api
 from agentops.sdk.core import TracingCore
-from agentops.sdk.processors import SpanProcessor
-from opentelemetry.instrumentation.utils import suppress_instrumentation
-from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace import TracerProvider as TracerProviderImpl
+from opentelemetry.trace import get_tracer_provider
 from opentelemetry.trace.status import StatusCode
 
 from agentlightning.instrumentation import instrument_all, uninstrument_all
 from agentlightning.store.base import LightningStore
 
-from .base import Tracer
+from .otel import LightningSpanProcessor, OtelTracer
 
 if TYPE_CHECKING:
     from agentops.integration.callbacks.langchain import LangchainCallbackHandler
@@ -29,7 +28,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-class AgentOpsTracer(Tracer):
+class AgentOpsTracer(OtelTracer):
     """Traces agent execution using AgentOps.
 
     This tracer provides functionality to capture execution details using the
@@ -67,9 +66,8 @@ class AgentOpsTracer(Tracer):
     def uninstrument(self, worker_id: int):
         uninstrument_all()
 
-    def init_worker(self, worker_id: int):
-        super().init_worker(worker_id)
-        logger.info(f"[Worker {worker_id}] Setting up tracer...")  # worker_id included in process name
+    def _initialize_tracer_provider(self, worker_id: int):
+        logger.info(f"[Worker {worker_id}] Setting up AgentOps tracer...")  # worker_id included in process name
 
         if self.instrument_managed:
             self.instrument(worker_id)
@@ -85,16 +83,9 @@ class AgentOpsTracer(Tracer):
 
         self._lightning_span_processor = LightningSpanProcessor()
 
-        try:
-            # new versions
-            instance = agentops.sdk.core.tracer
-            # TODO: The span processor cannot be deleted once added.
-            # This might be a problem if the tracer is entered and exited multiple times.
-            instance.provider.add_span_processor(self._lightning_span_processor)  # type: ignore
-        except AttributeError:
-            # old versions
-            instance = TracingCore.get_instance()  # type: ignore
-            instance._provider.add_span_processor(self._lightning_span_processor)  # type: ignore
+        # TODO: The span processor cannot be deleted once added.
+        # This might be a problem if the tracer is entered and exited multiple times.
+        self._get_tracer_provider().add_span_processor(self._lightning_span_processor)  # type: ignore
 
     def teardown_worker(self, worker_id: int) -> None:
         super().teardown_worker(worker_id)
@@ -111,7 +102,7 @@ class AgentOpsTracer(Tracer):
         store: Optional[LightningStore] = None,
         rollout_id: Optional[str] = None,
         attempt_id: Optional[str] = None,
-    ) -> AsyncGenerator[LightningSpanProcessor, None]:
+    ) -> AsyncGenerator[trace_api.Tracer, None]:
         """
         Starts a new tracing context. This should be used as a context manager.
 
@@ -122,12 +113,18 @@ class AgentOpsTracer(Tracer):
             attempt_id: Optional attempt ID to add the spans to.
 
         Yields:
-            The [`LightningSpanProcessor`][agentlightning.tracer.agentops.LightningSpanProcessor] instance to collect spans.
+            The OpenTelemetry tracer instance to collect spans.
         """
-        with self._trace_context_sync(
-            name=name, store=store, rollout_id=rollout_id, attempt_id=attempt_id
-        ) as processor:
-            yield processor
+        if store is not None:
+            warnings.warn(
+                "store is deprecated in favor of init_worker(). It will be removed in the future.",
+                DeprecationWarning,
+                stacklevel=3,
+            )
+        else:
+            store = self._store
+        with self._trace_context_sync(name=name, store=store, rollout_id=rollout_id, attempt_id=attempt_id) as tracer:
+            yield tracer
 
     @contextmanager
     def _trace_context_sync(
@@ -137,47 +134,52 @@ class AgentOpsTracer(Tracer):
         store: Optional[LightningStore] = None,
         rollout_id: Optional[str] = None,
         attempt_id: Optional[str] = None,
-    ) -> Iterator[LightningSpanProcessor]:
+    ) -> Iterator[trace_api.Tracer]:
         """Implementation of `trace_context` for synchronous execution."""
         if not self._lightning_span_processor:
             raise RuntimeError("LightningSpanProcessor is not initialized. Call init_worker() first.")
+        tracer_provider = self._get_tracer_provider()
 
         kwargs: dict[str, Any] = {}
         if name is not None:
             kwargs["trace_name"] = name
         elif rollout_id is not None:
             kwargs["trace_name"] = rollout_id
+        if store is not None and rollout_id is not None and attempt_id is not None:
+            if store.capabilities.get("otlp_traces", False) is True:
+                logger.debug(f"Tracing to LightningStore rollout_id={rollout_id}, attempt_id={attempt_id}")
+                self._enable_native_otlp_exporter(store, rollout_id, attempt_id)
+            else:
+                self._disable_native_otlp_exporter()
+            ctx = self._lightning_span_processor.with_context(store=store, rollout_id=rollout_id, attempt_id=attempt_id)
+            with ctx:
+                # AgentOps end_trace and start_trace must live inside the lightning span processor context.
+                # Otherwise some traces might not be recorded.
+                with self._agentops_trace_context(rollout_id, attempt_id, kwargs):
+                    yield trace_api.get_tracer(__name__, tracer_provider=tracer_provider)
+        elif store is None and rollout_id is None and attempt_id is None:
+            # TODO: Add tests to cover both paths
+            self._disable_native_otlp_exporter()
+            with self._lightning_span_processor:
+                with self._agentops_trace_context(None, None, kwargs):
+                    yield trace_api.get_tracer(__name__, tracer_provider=tracer_provider)
+        else:
+            raise ValueError("store, rollout_id, and attempt_id must be either all provided or all None")
+
+    @contextmanager
+    def _agentops_trace_context(self, rollout_id: Optional[str], attempt_id: Optional[str], kwargs: dict[str, Any]):
         trace = agentops.start_trace(**kwargs)
         status = StatusCode.OK  # type: ignore
         try:
-            if store is not None and rollout_id is not None and attempt_id is not None:
-                ctx = self._lightning_span_processor.with_context(
-                    store=store, rollout_id=rollout_id, attempt_id=attempt_id
-                )
-                with ctx as processor:
-                    yield processor
-            elif store is None and rollout_id is None and attempt_id is None:
-                with self._lightning_span_processor:
-                    yield self._lightning_span_processor
-            else:
-                raise ValueError("store, rollout_id, and attempt_id must be either all provided or all None")
+            yield
         except Exception as e:
+            # This will catch errors in user code.
             status = StatusCode.ERROR  # type: ignore
-            logger.error(f"Trace failed for rollout_id={rollout_id}, attempt_id={attempt_id}, error={e}")
+            logger.error(f"Trace failed for rollout_id={rollout_id}, attempt_id={attempt_id}: {e}")
+            raise  # should reraise the error here so that runner can handle it
         finally:
             agentops.end_trace(trace, end_state=status)  # type: ignore
 
-    def get_last_trace(self) -> List[ReadableSpan]:
-        """
-        Retrieves the raw list of captured spans from the most recent trace.
-
-        Returns:
-            A list of OpenTelemetry `ReadableSpan` objects.
-        """
-        if not self._lightning_span_processor:
-            raise RuntimeError("LightningSpanProcessor is not initialized. Call init_worker() first.")
-        return self._lightning_span_processor.spans()
-
     def get_langchain_handler(self, tags: List[str] | None = None) -> LangchainCallbackHandler:
         """
         Get the Langchain callback handler for integrating with Langchain.
@@ -204,135 +206,26 @@ class AgentOpsTracer(Tracer):
 
     get_langchain_callback_handler = get_langchain_handler  # alias
 
+    def _get_tracer_provider(self) -> TracerProviderImpl:
+        try:
+            # new versions
+            instance = agentops.sdk.core.tracer
+            if instance.provider is None:
+                raise RuntimeError("AgentOps TracerProvider is not initialized.")
 
-class LightningSpanProcessor(SpanProcessor):
-    """Span processor that subclasses OpenTelemetry's `SpanProcessor` and adds support to dump traces
-    to a [`LightningStore`][agentlightning.LightningStore].
-    """
+            if get_tracer_provider() is not instance.provider:
+                logger.error(
+                    "Mismatch between global singleton TracerProvider and AgentOps TracerProvider. "
+                    "AgentOps might not work properly."
+                )
 
-    def __init__(self):
-        self._spans: List[ReadableSpan] = []
+            if not isinstance(instance.provider, TracerProviderImpl):  # type: ignore
+                raise RuntimeError("Unsupported TracerProvider type for AgentOps instrumentation.")
 
-        # Store related context and states
-        self._store: Optional[LightningStore] = None
-        self._rollout_id: Optional[str] = None
-        self._attempt_id: Optional[str] = None
-        self._lock = threading.Lock()
-
-        # private asyncio loop running in a daemon thread
-        self._loop_ready = threading.Event()
-        self._loop: Optional[asyncio.AbstractEventLoop] = None
-        self._loop_thread = threading.Thread(target=self._loop_runner, name="otel-loop", daemon=True)
-        self._loop_thread.start()
-        self._loop_ready.wait()  # loop is ready
-
-    def _loop_runner(self):
-        loop = asyncio.new_event_loop()
-        self._loop = loop
-        asyncio.set_event_loop(loop)
-        self._loop_ready.set()
-        loop.run_forever()
-        loop.close()
-
-    def __enter__(self):
-        self._last_trace = None
-        self._spans = []
-        return self
-
-    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any):
-        self._store = None
-        self._rollout_id = None
-        self._attempt_id = None
-
-    def _await_in_loop(self, coro: Awaitable[Any], timeout: Optional[float] = None) -> Any:
-        # submit to the dedicated loop and wait synchronously
-        if self._loop is None:
-            raise RuntimeError("Loop is not initialized. This should not happen.")
-
-        # If already on the exporter loop thread, schedule and return immediately.
-        # ---------------------------------------------------------------------------
-        # WHY THIS CONDITIONAL EXISTS:
-        # In rare cases, span.end() is triggered from a LangchainCallbackHandler.__del__
-        # (or another finalizer) while the Python garbage collector is running on the
-        # *same thread* that owns our exporter event loop ("otel-loop").
-        #
-        # When that happens, on_end() executes on the exporter loop thread itself.
-        # If we were to call `asyncio.run_coroutine_threadsafe(...).result()` here,
-        # it would deadlock immediately — because the loop cannot both wait on and run
-        # the same coroutine. The Future stays pending forever and the loop stops
-        # processing scheduled callbacks.
-        #
-        # To avoid that self-deadlock, we detect when on_end() runs on the exporter
-        # loop thread. If so, we *schedule* the coroutine on the loop (fire-and-forget)
-        # instead of blocking with .result().
-        #
-        # This situation can occur because Python calls __del__ in whatever thread
-        # releases the last reference, which can easily be our loop thread if the
-        # object is dereferenced during loop._run_once().
-        # ---------------------------------------------------------------------------
-        if threading.current_thread() is self._loop_thread:
-            self._loop.call_soon_threadsafe(asyncio.create_task, coro)  # type: ignore
-            return None
-
-        fut = asyncio.run_coroutine_threadsafe(coro, self._loop)  # type: ignore
-        return fut.result(timeout=timeout)  # raises on error  # type: ignore
-
-    def shutdown(self) -> None:
-        if self._loop:
-            self._loop.call_soon_threadsafe(self._loop.stop)
-            self._loop_thread.join(timeout=5)
-            self._loop = None
-
-    def force_flush(self, timeout_millis: int = 30000) -> bool:
-        return True
-
-    def spans(self) -> List[ReadableSpan]:
-        """
-        Get the list of spans collected by this processor.
-        This is useful for debugging and testing purposes.
-
-        Returns:
-            List of ReadableSpan objects collected during tracing.
-        """
-        return self._spans
-
-    def with_context(self, store: LightningStore, rollout_id: str, attempt_id: str):
-        # simple context manager without nesting into asyncio
-        class _Ctx:
-            def __enter__(_):  # type: ignore
-                with self._lock:
-                    self._store, self._rollout_id, self._attempt_id = store, rollout_id, attempt_id
-                    self._last_trace = None
-                    self._spans = []
-                return self
-
-            def __exit__(_, exc_type, exc, tb):  # type: ignore
-                with self._lock:
-                    self._store = self._rollout_id = self._attempt_id = None
-
-        return _Ctx()
-
-    def on_end(self, span: ReadableSpan) -> None:
-        """
-        Process a span when it ends.
-
-        Args:
-            span: The span that has ended.
-        """
-        # Skip if span is not sampled
-        if not span.context or not span.context.trace_flags.sampled:
-            return
-
-        if self._store and self._rollout_id and self._attempt_id:
-            try:
-                # Submit add_otel_span to the event loop and wait for it to complete
-                with suppress_instrumentation():
-                    self._await_in_loop(
-                        self._store.add_otel_span(self._rollout_id, self._attempt_id, span),
-                        timeout=60.0,
-                    )
-            except Exception:
-                # log; on_end MUST NOT raise
-                logger.exception(f"Error adding span to store: {span.name}")
-
-        self._spans.append(span)
+            self._tracer_provider = instance.provider
+            return self._tracer_provider
+        except AttributeError:
+            # old versions
+            instance = TracingCore.get_instance()  # type: ignore
+            self._tracer_provider = instance._provider  # type: ignore
+            return self._tracer_provider  # type: ignore

agentlightning/tracer/base.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+from contextlib import contextmanager
 from typing import TYPE_CHECKING, Any, AsyncContextManager, Awaitable, Callable, ContextManager, List, Optional
 
 from opentelemetry.sdk.trace import ReadableSpan
@@ -51,6 +52,18 @@ class Tracer(ParallelWorkerBase):
     ```
     """
 
+    _store: Optional[LightningStore] = None
+
+    def init_worker(self, worker_id: int, store: Optional[LightningStore] = None) -> None:
+        """Initialize the tracer for a worker.
+
+        Args:
+            worker_id: The ID of the worker.
+            store: The store to add the spans to. If it's provided, traces will be added to the store when tracing.
+        """
+        super().init_worker(worker_id)
+        self._store = store
+
     def trace_context(
         self,
         name: Optional[str] = None,
@@ -67,11 +80,9 @@ class Tracer(ParallelWorkerBase):
         within the `with` block are collected and made available via
         [`get_last_trace`][agentlightning.Tracer.get_last_trace].
 
-        If a store is provided, the spans will be added to the store when tracing.
-
         Args:
             name: The name for the root span of this trace context.
-            store: The store to add the spans to.
+            store: The store to add the spans to. Deprecated in favor of passing store to init_worker().
             rollout_id: The rollout ID to add the spans to.
             attempt_id: The attempt ID to add the spans to.
         """
@@ -81,7 +92,6 @@ class Tracer(ParallelWorkerBase):
         self,
         name: Optional[str] = None,
         *,
-        store: Optional[LightningStore] = None,
         rollout_id: Optional[str] = None,
         attempt_id: Optional[str] = None,
     ) -> ContextManager[Any]:
@@ -138,3 +148,30 @@ class Tracer(ParallelWorkerBase):
         """
         logger.warning(f"{self.__class__.__name__} does not provide a LangChain callback handler.")
         return None
+
+    @contextmanager
+    def lifespan(self, store: Optional[LightningStore] = None):
+        """A context manager to manage the lifespan of the tracer.
+
+        This can be used to set up and tear down any necessary resources
+        for the tracer, useful for debugging purposes.
+
+        Args:
+            store: The store to add the spans to. If it's provided, traces will be added to the store when tracing.
+        """
+        has_init = False
+        has_init_worker = False
+        try:
+            self.init()
+            has_init = True
+
+            self.init_worker(0, store)
+            has_init_worker = True
+
+            yield
+
+        finally:
+            if has_init_worker:
+                self.teardown_worker(0)
+            if has_init:
+                self.teardown()

agentlightning/tracer/http.py

@@ -19,6 +19,8 @@ from opentelemetry.trace.span import (
     TraceState,
 )
 
+from agentlightning.store import LightningStore
+
 from .base import Tracer
 
 logger = logging.getLogger(__name__)
@@ -68,14 +70,15 @@ class HttpTracer(Tracer):
         self.subprocess_mode = subprocess_mode
         self.subprocess_timeout = subprocess_timeout
 
-    def init_worker(self, worker_id: int) -> None:
+    def init_worker(self, worker_id: int, store: Optional[LightningStore] = None) -> None:
         """
         Initialize the tracer in a worker process.
 
         Args:
             worker_id: The ID of the worker process.
+            store: The store to add the spans to.
         """
-        super().init_worker(worker_id)
+        super().init_worker(worker_id, store)
         logger.info(f"[Worker {worker_id}] HttpTracer initialized.")
 
     @asynccontextmanager

agentlightning/tracer/otel.py

@@ -2,16 +2,27 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
+import threading
+import warnings
 from contextlib import asynccontextmanager
-from typing import AsyncGenerator, List, Optional
+from typing import Any, AsyncGenerator, Awaitable, List, Optional
 
 import opentelemetry.trace as trace_api
-from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
+from agentops.sdk.core import BatchSpanProcessor
+from opentelemetry.instrumentation.utils import suppress_instrumentation
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace import TracerProvider as TracerProviderImpl
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
 
+from agentlightning.semconv import LightningResourceAttributes
 from agentlightning.store.base import LightningStore
+from agentlightning.utils.otel import get_tracer_provider
+from agentlightning.utils.otlp import LightningStoreOTLPExporter
 
-from .agentops import LightningSpanProcessor  # FIXME: This import should be from otel to agentops
 from .base import Tracer
 
 logger = logging.getLogger(__name__)
@@ -29,25 +40,43 @@ class OtelTracer(Tracer):
         # This provider is only initialized when the worker is initialized.
         self._tracer_provider: Optional[TracerProvider] = None
         self._lightning_span_processor: Optional[LightningSpanProcessor] = None
+        self._simple_span_processor: Optional[SimpleSpanProcessor] = None
+        self._otlp_span_exporter: Optional[LightningStoreOTLPExporter] = None
         self._initialized: bool = False
 
-    def init_worker(self, worker_id: int):
-        super().init_worker(worker_id)
+    def init_worker(self, worker_id: int, store: Optional[LightningStore] = None):
+        super().init_worker(worker_id, store)
+        self._initialize_tracer_provider(worker_id)
+
+    def _initialize_tracer_provider(self, worker_id: int):
         logger.info(f"[Worker {worker_id}] Setting up OpenTelemetry tracer...")
 
         if self._initialized:
-            logger.error("Tracer provider is already initialized. OpenTelemetry may not work as expected.")
+            logger.info(f"[Worker {worker_id}] Tracer provider is already initialized. Skipping initialization.")
+            return
 
-        tracer_provider = TracerProvider()
-        trace_api.set_tracer_provider(tracer_provider)
+        try:
+            get_tracer_provider()
+            logger.error(
+                f"[Worker {worker_id}] Tracer provider is already initialized but not by OtelTracer. OpenTelemetry may not work as expected."
+            )
+        except RuntimeError:
+            logger.debug(f"[Worker {worker_id}] Tracer provider is not initialized by OtelTracer. Initializing it now.")
+
+        self._tracer_provider = TracerProvider()
+        trace_api.set_tracer_provider(self._tracer_provider)
         self._lightning_span_processor = LightningSpanProcessor()
-        tracer_provider.add_span_processor(self._lightning_span_processor)
+        self._tracer_provider.add_span_processor(self._lightning_span_processor)
+        self._otlp_span_exporter = LightningStoreOTLPExporter()
+        self._simple_span_processor = SimpleSpanProcessor(self._otlp_span_exporter)
+        self._tracer_provider.add_span_processor(self._simple_span_processor)
         self._initialized = True
 
+        logger.info(f"[Worker {worker_id}] OpenTelemetry tracer provider initialized.")
+
     def teardown_worker(self, worker_id: int):
         super().teardown_worker(worker_id)
-        logger.info(f"[Worker {worker_id}] Tearing down OpenTelemetry tracer...")
-        self._tracer_provider = None
+        logger.info(f"[Worker {worker_id}] Tearing down OpenTelemetry tracer does NOT remove the tracer provider.")
 
     @asynccontextmanager
     async def trace_context(
@@ -57,7 +86,7 @@ class OtelTracer(Tracer):
         store: Optional[LightningStore] = None,
         rollout_id: Optional[str] = None,
         attempt_id: Optional[str] = None,
-    ) -> AsyncGenerator[LightningSpanProcessor, None]:
+    ) -> AsyncGenerator[trace_api.Tracer, None]:
         """
         Starts a new tracing context. This should be used as a context manager.
 
@@ -68,20 +97,37 @@ class OtelTracer(Tracer):
             attempt_id: Optional attempt ID to add the spans to.
 
         Yields:
-            The LightningSpanProcessor instance to collect spans.
+            The OpenTelemetry tracer instance to collect spans.
         """
         if not self._lightning_span_processor:
             raise RuntimeError("LightningSpanProcessor is not initialized. Call init_worker() first.")
 
-        if store is not None and rollout_id is not None and attempt_id is not None:
-            ctx = self._lightning_span_processor.with_context(store=store, rollout_id=rollout_id, attempt_id=attempt_id)
-            with ctx as processor:
-                yield processor
-        elif store is None and rollout_id is None and attempt_id is None:
-            with self._lightning_span_processor:
-                yield self._lightning_span_processor
+        if store is not None:
+            warnings.warn(
+                "store is deprecated in favor of init_worker(). It will be removed in the future.",
+                DeprecationWarning,
+                stacklevel=3,
+            )
         else:
-            raise ValueError("store, rollout_id, and attempt_id must be either all provided or all None")
+            store = self._store
+
+        if rollout_id is not None and attempt_id is not None:
+            if store is None:
+                raise ValueError("store is required to be initialized when rollout_id and attempt_id are provided")
+            if store.capabilities.get("otlp_traces", False) is True:
+                logger.debug(f"Tracing to LightningStore rollout_id={rollout_id}, attempt_id={attempt_id}")
+                self._enable_native_otlp_exporter(store, rollout_id, attempt_id)
+            else:
+                self._disable_native_otlp_exporter()
+            ctx = self._lightning_span_processor.with_context(store=store, rollout_id=rollout_id, attempt_id=attempt_id)
+            with ctx:
+                yield trace_api.get_tracer(__name__, tracer_provider=self._tracer_provider)
+        elif rollout_id is None and attempt_id is None:
+            self._disable_native_otlp_exporter()
+            with self._lightning_span_processor:
+                yield trace_api.get_tracer(__name__, tracer_provider=self._tracer_provider)
+        else:
+            raise ValueError("rollout_id and attempt_id must be either all provided or all None")
 
     def get_last_trace(self) -> List[ReadableSpan]:
         """
@@ -93,3 +139,246 @@ class OtelTracer(Tracer):
         if not self._lightning_span_processor:
             raise RuntimeError("LightningSpanProcessor is not initialized. Call init_worker() first.")
         return self._lightning_span_processor.spans()
+
+    def _get_tracer_provider(self) -> TracerProviderImpl:
+        if self._tracer_provider is None:
+            raise RuntimeError("TracerProvider is not initialized. Call init_worker() first.")
+        return self._tracer_provider
+
+    def _enable_native_otlp_exporter(self, store: LightningStore, rollout_id: str, attempt_id: str):
+        tracer_provider = self._get_tracer_provider()
+        active_span_processor = tracer_provider._active_span_processor  # pyright: ignore[reportPrivateUsage]
+
+        # Override the resources so that the server knows where the request comes from.
+        tracer_provider._resource = tracer_provider._resource.merge(  # pyright: ignore[reportPrivateUsage]
+            Resource.create(
+                {
+                    LightningResourceAttributes.ROLLOUT_ID.value: rollout_id,
+                    LightningResourceAttributes.ATTEMPT_ID.value: attempt_id,
+                }
+            )
+        )
+        instrumented = False
+        candidates: List[str] = []
+        for processor in active_span_processor._span_processors:  # pyright: ignore[reportPrivateUsage]
+            if isinstance(processor, LightningSpanProcessor):
+                # We don't need the LightningSpanProcessor any more.
+                logger.debug("LightningSpanProcessor already present in TracerProvider, disabling it.")
+                processor.disable_store_submission = True
+            elif isinstance(processor, (SimpleSpanProcessor, BatchSpanProcessor)):
+                # Instead, we rely on the OTLPSpanExporter to send spans to the store.
+                if isinstance(processor.span_exporter, LightningStoreOTLPExporter):
+                    processor.span_exporter.enable_store_otlp(store.otlp_traces_endpoint(), rollout_id, attempt_id)
+                    logger.debug(f"Set LightningStoreOTLPExporter endpoint to {store.otlp_traces_endpoint()}")
+                    instrumented = True
+                else:
+                    candidates.append(
+                        f"{processor.__class__.__name__} with {processor.span_exporter.__class__.__name__}"
+                    )
+            else:
+                candidates.append(f"{processor.__class__.__name__}")
+
+        if not instrumented:
+            raise RuntimeError(
+                "Failed to enable native OTLP exporter: no BatchSpanProcessor or SimpleSpanProcessor with "
+                "LightningStoreOTLPExporter found in TracerProvider. Please try using a non-OTLP store."
+                "Candidates are: " + ", ".join(candidates)
+            )
+
+    def _disable_native_otlp_exporter(self):
+        tracer_provider = self._get_tracer_provider()
+        active_span_processor = tracer_provider._active_span_processor  # pyright: ignore[reportPrivateUsage]
+        tracer_provider._resource = tracer_provider._resource.merge(  # pyright: ignore[reportPrivateUsage]
+            Resource.create(
+                {
+                    LightningResourceAttributes.ROLLOUT_ID.value: "",
+                    LightningResourceAttributes.ATTEMPT_ID.value: "",
+                }
+            )
+        )  # reset resource
+        for processor in active_span_processor._span_processors:  # pyright: ignore[reportPrivateUsage]
+            if isinstance(processor, LightningSpanProcessor):
+                # We will be in need of the LightningSpanProcessor again.
+                logger.debug("Enabling LightningSpanProcessor in TracerProvider.")
+                processor.disable_store_submission = False
+
+
+class LightningSpanProcessor(SpanProcessor):
+    """Span processor that subclasses OpenTelemetry's `SpanProcessor` and adds support to dump traces
+    to a [`LightningStore`][agentlightning.LightningStore].
+
+    It serves two purposes:
+
+    1. Records all the spans in a local buffer.
+    2. Submits the spans to the event loop to be added to the store.
+    """
+
+    def __init__(self, disable_store_submission: bool = False):
+        self._disable_store_submission: bool = disable_store_submission
+        self._spans: List[ReadableSpan] = []
+
+        # Store related context and states
+        self._store: Optional[LightningStore] = None
+        self._rollout_id: Optional[str] = None
+        self._attempt_id: Optional[str] = None
+        self._lock = threading.Lock()
+
+        # private asyncio loop running in a daemon thread
+        self._loop_ready = threading.Event()
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._loop_thread: Optional[threading.Thread] = None
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}("
+            + f"disable_store_submission={self.disable_store_submission}, "
+            + f"store={self.store!r}, "
+            + f"rollout_id={self.rollout_id!r}, "
+            + f"attempt_id={self.attempt_id!r})"
+        )
+
+    @property
+    def store(self) -> Optional[LightningStore]:
+        """The store to submit the spans to."""
+        return self._store
+
+    @property
+    def rollout_id(self) -> Optional[str]:
+        """The rollout ID to submit the spans to."""
+        return self._rollout_id
+
+    @property
+    def attempt_id(self) -> Optional[str]:
+        """The attempt ID to submit the spans to."""
+        return self._attempt_id
+
+    @property
+    def disable_store_submission(self) -> bool:
+        """Whether to disable submitting spans to the store."""
+        return self._disable_store_submission
+
+    @disable_store_submission.setter
+    def disable_store_submission(self, value: bool) -> None:
+        self._disable_store_submission = value
+
+    def _ensure_loop(self) -> None:
+        if self._loop_thread is None or self._loop is None:
+            self._loop_ready.clear()
+            self._loop_thread = threading.Thread(target=self._loop_runner, name="otel-loop", daemon=True)
+            self._loop_thread.start()
+            self._loop_ready.wait()  # loop is ready
+
+    def _loop_runner(self):
+        loop = asyncio.new_event_loop()
+        self._loop = loop
+        asyncio.set_event_loop(loop)
+        self._loop_ready.set()
+        loop.run_forever()
+        loop.close()
+
+    def __enter__(self):
+        self._last_trace = None
+        self._spans = []
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any):
+        self._store = None
+        self._rollout_id = None
+        self._attempt_id = None
+
+    def _await_in_loop(self, coro: Awaitable[Any], timeout: Optional[float] = None) -> Any:
+        # submit to the dedicated loop and wait synchronously
+        self._ensure_loop()
+        if self._loop is None:
+            raise RuntimeError("Loop is not initialized. This should not happen.")
+
+        # If already on the exporter loop thread, schedule and return immediately.
+        # ---------------------------------------------------------------------------
+        # WHY THIS CONDITIONAL EXISTS:
+        # In rare cases, span.end() is triggered from a LangchainCallbackHandler.__del__
+        # (or another finalizer) while the Python garbage collector is running on the
+        # *same thread* that owns our exporter event loop ("otel-loop").
+        #
+        # When that happens, on_end() executes on the exporter loop thread itself.
+        # If we were to call `asyncio.run_coroutine_threadsafe(...).result()` here,
+        # it would deadlock immediately — because the loop cannot both wait on and run
+        # the same coroutine. The Future stays pending forever and the loop stops
+        # processing scheduled callbacks.
+        #
+        # To avoid that self-deadlock, we detect when on_end() runs on the exporter
+        # loop thread. If so, we *schedule* the coroutine on the loop (fire-and-forget)
+        # instead of blocking with .result().
+        #
+        # This situation can occur because Python calls __del__ in whatever thread
+        # releases the last reference, which can easily be our loop thread if the
+        # object is dereferenced during loop._run_once().
+        # ---------------------------------------------------------------------------
+        if threading.current_thread() is self._loop_thread:
+            self._loop.call_soon_threadsafe(asyncio.create_task, coro)  # type: ignore
+            return None
+
+        fut = asyncio.run_coroutine_threadsafe(coro, self._loop)  # type: ignore
+        return fut.result(timeout=timeout)  # raises on error  # type: ignore
+
+    def shutdown(self) -> None:
+        if self._loop:
+            self._loop.call_soon_threadsafe(self._loop.stop)
+            self._loop = None
+        if self._loop_thread:
+            self._loop_thread.join(timeout=5)
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        return True
+
+    def spans(self) -> List[ReadableSpan]:
+        """
+        Get the list of spans collected by this processor.
+        This is useful for debugging and testing purposes.
+
+        Returns:
+            List of ReadableSpan objects collected during tracing.
+        """
+        return self._spans
+
+    def with_context(self, store: LightningStore, rollout_id: str, attempt_id: str):
+        # simple context manager without nesting into asyncio
+        class _Ctx:
+            def __enter__(_):  # type: ignore
+                # Use _ instead of self to avoid shadowing the instance method.
+                with self._lock:
+                    self._store, self._rollout_id, self._attempt_id = store, rollout_id, attempt_id
+                    self._last_trace = None
+                    self._spans = []
+                return self
+
+            def __exit__(_, exc_type, exc, tb):  # type: ignore
+                with self._lock:
+                    self._store = self._rollout_id = self._attempt_id = None
+
+        return _Ctx()
+
+    def on_end(self, span: ReadableSpan) -> None:
+        """
+        Process a span when it ends.
+
+        Args:
+            span: The span that has ended.
+        """
+        # Skip if span is not sampled
+        if not span.context or not span.context.trace_flags.sampled:
+            return
+
+        if not self._disable_store_submission and self._store and self._rollout_id and self._attempt_id:
+            try:
+                # Submit add_otel_span to the event loop and wait for it to complete
+                with suppress_instrumentation():
+                    self._ensure_loop()
+                    self._await_in_loop(
+                        self._store.add_otel_span(self._rollout_id, self._attempt_id, span),
+                        timeout=60.0,
+                    )
+            except Exception:
+                # log; on_end MUST NOT raise
+                logger.exception(f"Error adding span to store: {span.name}")
+
+        self._spans.append(span)

agentlightning/trainer/trainer.py

@@ -152,6 +152,13 @@ class Trainer(TrainerLegacy):
         # super().__init__() will call TrainerLegacy's initialization, which is not intended.
         self.worker_id: Optional[int] = None
 
+        if dev:
+            logger.warning(
+                "Trainer(dev=True) is deprecated and will be removed in future versions. "
+                "Please use Trainer.dev(...) instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
         self._dev = dev
         self.daemon = daemon
         self._client: AgentLightningClient | None = None  # Will be initialized in fit or fit_v0
@@ -213,10 +220,6 @@ class Trainer(TrainerLegacy):
         # We might be able to support a list of resources in future.
         self.initial_resources = initial_resources
 
-        # The active store for the current execution context
-        self.store = self._make_store(store)
-        self.runner = self._make_runner(runner)
-
         self.port = port
 
         self.strategy = self._make_strategy(
@@ -224,6 +227,11 @@ class Trainer(TrainerLegacy):
             n_runners=self.n_runners,
             port=port,
         )
+
+        # The active store for the current execution context
+        self.store = self._make_store(store, self.strategy)
+        self.runner = self._make_runner(runner)
+
         if hasattr(self.strategy, "n_runners"):
             strategy_runners = getattr(self.strategy, "n_runners")
             if isinstance(strategy_runners, int) and strategy_runners > 0:
@@ -282,13 +290,19 @@ class Trainer(TrainerLegacy):
             type_error_fmt="Adapter factory returned {type_name}, which is not a TraceAdapter subclass.",
         )
 
-    def _make_store(self, store: ComponentSpec[LightningStore]) -> LightningStore:
-        """Resolve the store implementation backing rollouts, attempts, spans, and resources."""
+    def _make_store(self, store: ComponentSpec[LightningStore], strategy: ExecutionStrategy) -> LightningStore:
+        """Resolve the store implementation backing rollouts, attempts, spans, and resources.
+
+        By default, it's always a in-memory store. If using a client/server execution strategy,
+        the in-memory store will be initialized in a thread-safe manner.
+        """
+        is_client_server = isinstance(strategy, ClientServerExecutionStrategy)
+        default_store_factory = lambda: InMemoryLightningStore(thread_safe=is_client_server)
         return build_component(
             store,
             expected_type=LightningStore,
             spec_name="store",
-            default_factory=InMemoryLightningStore,
+            default_factory=default_store_factory,
             invalid_spec_error_fmt="Invalid store type: {actual_type}. Expected LightningStore, str, dict, or None.",
             type_error_fmt="Store factory returned {type_name}, which is not a LightningStore subclass.",
         )

agentlightning/types/core.py

@@ -10,14 +10,19 @@ from typing import (
     Callable,
     Dict,
     Generic,
+    Iterator,
     List,
     Literal,
+    Mapping,
     Optional,
     Protocol,
+    Sequence,
     SupportsIndex,
+    TypedDict,
     TypeVar,
     Union,
     cast,
+    overload,
 )
 
 from opentelemetry.sdk.trace import ReadableSpan
@@ -48,7 +53,14 @@ __all__ = [
     "Rollout",
     "Attempt",
     "AttemptedRollout",
+    "EnqueueRolloutRequest",
     "Hook",
+    "Worker",
+    "WorkerStatus",
+    "PaginatedResult",
+    "FilterOptions",
+    "SortOptions",
+    "FilterField",
 ]
 
 T_co = TypeVar("T_co", covariant=True)
@@ -200,6 +212,50 @@ class AttemptedRollout(Rollout):
         return self
 
 
+class EnqueueRolloutRequest(BaseModel):
+    """Payload describing a rollout to be queued via [`enqueue_rollout`][agentlightning.LightningStore.enqueue_rollout].
+
+    A subset of fields from [`Rollout`][agentlightning.Rollout] used for queuing new rollouts.
+    """
+
+    input: TaskInput
+    """Task input used to generate the rollout."""
+    mode: Optional[RolloutMode] = None
+    """Execution mode such as `"train"`, `"val"` or `"test"`. See [`RolloutMode`][agentlightning.RolloutMode]."""
+    resources_id: Optional[str] = None
+    """Identifier of the resources required to execute the rollout."""
+    config: Optional[RolloutConfig] = None
+    """Retry and timeout configuration associated with the rollout."""
+    metadata: Optional[Dict[str, Any]] = None
+    """Additional metadata attached to the rollout."""
+
+
+WorkerStatus = Literal["idle", "busy", "unknown"]
+
+
+class Worker(BaseModel):
+    """Worker information. This is actually the same as Runner info."""
+
+    worker_id: str
+    """The ID of the worker."""
+    status: WorkerStatus = "unknown"
+    """The status of the worker."""
+    heartbeat_stats: Optional[Dict[str, Any]] = None
+    """Statistics about the worker's heartbeat."""
+    last_heartbeat_time: Optional[float] = None
+    """The last time when the worker has reported the stats."""
+    last_dequeue_time: Optional[float] = None
+    """The last time when the worker has tried to dequeue a rollout."""
+    last_busy_time: Optional[float] = None
+    """The last time when the worker has started an attempt and became busy."""
+    last_idle_time: Optional[float] = None
+    """The last time when the worker has triggered the end of an attempt and became idle."""
+    current_rollout_id: Optional[str] = None
+    """The ID of the current rollout that the worker is processing."""
+    current_attempt_id: Optional[str] = None
+    """The ID of the current attempt that the worker is processing."""
+
+
 TaskInput = Any
 """Task input type. Accepts arbitrary payloads."""
 
@@ -393,3 +449,104 @@ class Hook(ParallelWorkerBase):
         Subclasses can override this method for cleanup or additional
         logging. By default, this is a no-op.
         """
+
+
+class FilterField(TypedDict, total=False):
+    """An operator dict for a single field."""
+
+    exact: Any
+    within: Sequence[Any]
+    contains: str
+
+
+FilterOptions = Mapping[
+    Union[str, Literal["_aggregate", "_must"]],
+    Union[FilterField, Literal["and", "or"], Mapping[str, FilterField]],
+]
+"""A mapping of field name -> operator dict.
+
+Each operator dict can contain:
+
+- "exact": value for exact equality.
+- "within": iterable of allowed values.
+- "contains": substring to search for in string fields.
+
+The filter can also have a special field called "_aggregate" that can be used to specify the logic
+to combine the results of the filters:
+
+- "and": all conditions must match. This is the default value if not specified.
+- "or": at least one condition must match.
+
+All conditions within a field and between different fields are
+stored in a unified pool and combined using `_aggregate`.
+
+The filter can also have a special group called "_must", which is a mapping of filters that must all match,
+no matter whether the aggregate logic is "and" or "or".
+
+Example:
+
+```json
+{
+    "_aggregate": "or",
+    "_must": {
+        "city": {"exact": "New York"},
+        "timezone": {"within": ["America/New_York", "America/Los_Angeles"]},
+    },
+    "status": {"exact": "active"},
+    "id": {"within": [1, 2, 3]},
+    "name": {"contains": "foo"},
+}
+```
+"""
+
+
+class SortOptions(TypedDict):
+    """Options for sorting the collection."""
+
+    name: str
+    """The name of the field to sort by."""
+    order: Literal["asc", "desc"]
+    """The order to sort by."""
+
+
+T_item = TypeVar("T_item")
+
+
+class PaginatedResult(BaseModel, Sequence[T_item]):
+    """Result of a paginated query.
+
+    Behaves like a sequence, but also carries pagination metadata (limit, offset, total).
+    """
+
+    items: Sequence[T_item]
+    """Items in the result."""
+    limit: int
+    """Limit of the result."""
+    offset: int
+    """Offset of the result."""
+    total: int
+    """Total number of items in the collection."""
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    @overload
+    def __getitem__(self, index: int) -> T_item: ...
+
+    @overload
+    def __getitem__(self, index: slice) -> Sequence[T_item]: ...
+
+    def __getitem__(self, index: Union[int, slice]) -> Union[T_item, Sequence[T_item]]:
+        return self.items[index]
+
+    # Overriding __iter__ enables list(paginated_result) to work as expected,
+    # but changes Pydantic's default dict iteration behavior (which would otherwise
+    # iterate over field names).
+    def __iter__(self) -> Iterator[T_item]:  # type: ignore
+        return iter(self.items)
+
+    def __repr__(self) -> str:
+        first_item_repr = repr(self.items[0]) if self.items else "empty"
+        items_repr = f"[{first_item_repr}, ...]" if len(self.items) > 1 else first_item_repr
+        slice_repr = f"{self.offset}:" if self.limit == -1 else f"{self.offset}:{self.offset + self.limit}"
+        return f"<PaginatedResult ({slice_repr} of {self.total}) {items_repr}>"

agentlightning/types/tracer.py

@@ -16,6 +16,8 @@ from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
 from opentelemetry.trace.status import Status as OtelStatus
 from pydantic import BaseModel, ConfigDict
 
+from agentlightning.semconv import AGL_VIRTUAL
+
 __all__ = [
     "AttributeValue",
     "Attributes",
@@ -379,7 +381,7 @@ class Span(BaseModel):
                 is_remote=False,
                 trace_state={},
             ),
-            name=name or SpanNames.VIRTUAL.value,
+            name=name or AGL_VIRTUAL,
             resource=resource or OtelResource(attributes={}, schema_url=""),
             attributes=attributes,
             status=TraceStatus(status_code="OK"),
@@ -399,7 +401,7 @@ class Span(BaseModel):
 
 
 class SpanNames(str, Enum):
-    """Enumerated span names recognised by Agent-lightning."""
+    """Enumerated span names recognised by Agent-lightning. Deprecated in favor of [semconv][agentlightning.semconv]."""
 
     REWARD = "agentlightning.reward"
     """The name of the reward span."""
@@ -411,10 +413,16 @@ class SpanNames(str, Enum):
     """The name of the exception span."""
     VIRTUAL = "agentlightning.virtual"
     """The name of the virtual span. It represents derived spans without concrete operations."""
+    ROLLOUT_ID = "agentlightning.rollout_id"
+    """The name of the rollout ID."""
+    ATTEMPT_ID = "agentlightning.attempt_id"
+    """The name of the attempt ID."""
+    SPAN_SEQUENCE_ID = "agentlightning.span_sequence_id"
+    """The name of the span sequence ID."""
 
 
 class SpanAttributeNames(str, Enum):
-    """Canonical attribute names written by Agent Lightning emitters."""
+    """Canonical attribute names written by Agent Lightning emitters. Deprecated in favor of [semconv][agentlightning.semconv]."""
 
     MESSAGE = "message"
     """The name of the message attribute."""

agentlightning/utils/metrics.py

@@ -0,0 +1,873 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Metrics abstraction with explicit registration and several backends.
+
+It provides:
+
+- MetricsBackend: Abstract interface for registering and recording metrics.
+- ConsoleMetricsBackend: In-process backend with sliding-window
+  aggregations (rate, P50, P95, P99) logged to stdout.
+- PrometheusMetricsBackend: Thin wrapper around prometheus_client.
+- MultiMetricsBackend: Fan-out backend that forwards calls to multiple underlying backends.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import tempfile
+import threading
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple
+
+if TYPE_CHECKING:
+    from prometheus_client import CollectorRegistry
+
+LabelDict = Dict[str, str]
+LabelKey = Tuple[Tuple[str, str], ...]  # normalized, sorted (key, value) pairs
+
+logger = logging.getLogger(__name__)
+
+
+def _validate_labels(
+    kind: str,
+    metric_name: str,
+    labels: LabelDict,
+    expected_names: Tuple[str, ...],
+) -> LabelKey:
+    """Validates label keys against the metric definition.
+
+    Args:
+        kind: Metric kind for error messages ("counter" or "histogram").
+        metric_name: Metric name.
+        labels: Provided label dictionary.
+        expected_names: Expected label names as a tuple.
+
+    Returns:
+        A tuple of (key, value) pairs sorted by registered label order.
+
+    Raises:
+        ValueError: If label keys do not match expected_names.
+    """
+
+    label_items: List[Tuple[str, str]] = []
+    for label_name in expected_names:
+        if label_name not in labels:
+            raise ValueError(f"Label '{label_name}' is required for {kind.capitalize()} '{metric_name}'.")
+        label_items.append((label_name, labels[label_name]))
+
+    return tuple(label_items)
+
+
+def _normalize_label_names(label_names: Optional[Sequence[str]]) -> Tuple[str, ...]:
+    """Normalizes label names into a canonical tuple.
+
+    Args:
+        label_names: Iterable of label names or None.
+
+    Returns:
+        A tuple of label names sorted alphabetically.
+    """
+    if not label_names:
+        return ()
+    return tuple(sorted(label_names))
+
+
+@dataclass(frozen=True)
+class _CounterDef:
+    """Definition of a registered counter metric."""
+
+    name: str
+    label_names: Tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class _HistogramDef:
+    """Definition of a registered histogram metric."""
+
+    name: str
+    label_names: Tuple[str, ...]
+    buckets: Tuple[float, ...]
+
+
+@dataclass
+class _CounterState:
+    """Runtime state of a counter metric group (for console backend)."""
+
+    timestamps: List[float]
+    amounts: List[float]
+
+
+@dataclass
+class _HistogramState:
+    """Runtime state of a histogram metric group (for console backend)."""
+
+    timestamps: List[float]
+    values: List[float]
+
+
+class MetricsBackend:
+    """Abstract base class for metrics backends."""
+
+    def register_counter(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+    ) -> None:
+        """Registers a counter metric.
+
+        Args:
+            name: Metric name.
+            label_names: List of label names. Order is not important.
+
+        Raises:
+            ValueError: If the metric is already registered with a different
+                type or label set.
+        """
+        raise NotImplementedError()
+
+    def register_histogram(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+        buckets: Optional[Sequence[float]] = None,
+    ) -> None:
+        """Registers a histogram metric.
+
+        Args:
+            name: Metric name.
+            label_names: List of label names. Order is not important.
+            buckets: Bucket boundaries (exclusive upper bounds). If None, the
+                backend may choose defaults.
+
+        Raises:
+            ValueError: If the metric is already registered with a different
+                type or label set.
+        """
+        raise NotImplementedError()
+
+    def inc_counter(
+        self,
+        name: str,
+        amount: float = 1.0,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Increments a registered counter.
+
+        Args:
+            name: Metric name (must be registered as a counter).
+            amount: Increment amount.
+            labels: Label values.
+
+        Raises:
+            ValueError: If the metric is not registered, has the wrong type,
+                or label keys do not match the registered label names.
+        """
+        raise NotImplementedError()
+
+    def observe_histogram(
+        self,
+        name: str,
+        value: float,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Records an observation for a registered histogram.
+
+        Args:
+            name: Metric name (must be registered as a histogram).
+            value: Observed value.
+            labels: Label values.
+
+        Raises:
+            ValueError: If the metric is not registered, has the wrong type,
+                or label keys do not match the registered label names.
+        """
+        raise NotImplementedError()
+
+
+class ConsoleMetricsBackend(MetricsBackend):
+    """Console backend with sliding-window aggregations and label grouping.
+
+    This backend:
+
+    * Requires explicit metric registration.
+    * Stores timestamped events per (metric_name, labels) key.
+    * Computes rate and percentiles (P50, P95, P99) over a sliding time window.
+    * Uses a single global logging decision: when logging is triggered, it
+      logs all metric groups, not just the one being updated.
+
+    Rate is always per second.
+
+    Label grouping: When logging, labels are truncated to the first `group_level` label
+    pairs (according to sorted label key order). For example:
+
+        labels = {"method": "GET", "path": "/", "status": "200"}
+        group_level = 2 -> logged labels {"method": "GET", "path": "/"}
+
+    If `group_level` is None or < 1, all labels are logged.
+
+    Thread-safety: A single lock protects shared state mutation, pruning, and snapshotting.
+    Percentile computation, formatting, and printing are done after releasing the lock.
+    """
+
+    def __init__(
+        self,
+        window_seconds: Optional[float] = 60.0,
+        log_interval_seconds: float = 5.0,
+        group_level: Optional[int] = None,
+    ) -> None:
+        """Initializes ConsoleMetricsBackend.
+
+        Args:
+            window_seconds: Sliding window size (in seconds) used when computing
+                rate and percentiles. If None, all in-memory events are used.
+            log_interval_seconds: Minimum time (in seconds) between log bursts.
+                When the interval elapses, the next metric event triggers a
+                snapshot and logging of all metrics.
+            group_level: Label grouping depth. When logging, only the first
+                `group_level` labels (sorted by key) are included. If None or
+                < 1, all labels are included.
+        """
+        self.window_seconds = window_seconds
+        self.log_interval_seconds = log_interval_seconds
+        self.group_level = group_level
+
+        self._counters: Dict[str, _CounterDef] = {}
+        self._histograms: Dict[str, _HistogramDef] = {}
+
+        # Runtime state keyed by (metric_name, label_key)
+        self._counter_state: Dict[Tuple[str, LabelKey], _CounterState] = {}
+        self._hist_state: Dict[Tuple[str, LabelKey], _HistogramState] = {}
+
+        # Global last log time (for all metrics)
+        self._last_log_time: Optional[float] = None
+
+        self._lock = threading.Lock()
+
+    def register_counter(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+    ) -> None:
+        """Registers a counter metric.
+
+        See base class for argument documentation.
+        """
+        label_tuple = _normalize_label_names(label_names)
+        with self._lock:
+            existing_counter = self._counters.get(name)
+            existing_hist = self._histograms.get(name)
+
+            if existing_hist is not None:
+                raise ValueError(f"Metric '{name}' already registered as histogram.")
+
+            if existing_counter is not None:
+                if existing_counter.label_names != label_tuple:
+                    raise ValueError(
+                        f"Counter '{name}' already registered with labels "
+                        f"{existing_counter.label_names}, got {label_tuple}."
+                    )
+                return
+
+            self._counters[name] = _CounterDef(name=name, label_names=label_tuple)
+
+    def register_histogram(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+        buckets: Optional[Sequence[float]] = None,
+    ) -> None:
+        """Registers a histogram metric.
+
+        See base class for argument documentation.
+        """
+        label_tuple = _normalize_label_names(label_names)
+        if buckets is None:
+            bucket_tuple: Tuple[float, ...] = (0.1, 0.2, 0.5, 1.0, 2.0)
+        else:
+            bucket_tuple = tuple(buckets)
+
+        with self._lock:
+            existing_counter = self._counters.get(name)
+            existing_hist = self._histograms.get(name)
+
+            if existing_counter is not None:
+                raise ValueError(f"Metric '{name}' already registered as counter.")
+
+            if existing_hist is not None:
+                if existing_hist.label_names != label_tuple or existing_hist.buckets != bucket_tuple:
+                    raise ValueError(
+                        f"Histogram '{name}' already registered with "
+                        f"labels={existing_hist.label_names}, "
+                        f"buckets={existing_hist.buckets}."
+                    )
+                return
+
+            self._histograms[name] = _HistogramDef(
+                name=name,
+                label_names=label_tuple,
+                buckets=bucket_tuple,
+            )
+
+    def inc_counter(
+        self,
+        name: str,
+        amount: float = 1.0,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Increments a registered counter metric.
+
+        See base class for behavior and error conditions.
+        """
+        now = time.time()
+        labels = labels or {}
+
+        definition = self._counters.get(name)
+        if definition is None:
+            raise ValueError(f"Counter '{name}' is not registered.")
+
+        label_key = _validate_labels("counter", name, labels, definition.label_names)
+        state_key = (name, label_key)
+
+        with self._lock:
+            state = self._counter_state.get(state_key)
+            if state is None:
+                state = _CounterState(timestamps=[], amounts=[])
+                self._counter_state[state_key] = state
+
+            state.timestamps.append(now)
+            state.amounts.append(amount)
+            self._prune_events(state.timestamps, state.amounts, now)
+
+            should_log = self._should_log_locked(now)
+            if should_log:
+                counter_snaps, hist_snaps = self._snapshot_locked(now)
+                snapshot_time = now
+            else:
+                counter_snaps = hist_snaps = []
+                snapshot_time = now
+
+        if should_log and (counter_snaps or hist_snaps):
+            self._log_snapshot(counter_snaps, hist_snaps, snapshot_time)
+
+    def observe_histogram(
+        self,
+        name: str,
+        value: float,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Records an observation for a registered histogram metric.
+
+        See base class for behavior and error conditions.
+        """
+        now = time.time()
+        labels = labels or {}
+
+        definition = self._histograms.get(name)
+        if definition is None:
+            raise ValueError(f"Histogram '{name}' is not registered.")
+
+        label_key = _validate_labels("histogram", name, labels, definition.label_names)
+        state_key = (name, label_key)
+
+        with self._lock:
+            state = self._hist_state.get(state_key)
+            if state is None:
+                state = _HistogramState(timestamps=[], values=[])
+                self._hist_state[state_key] = state
+
+            state.timestamps.append(now)
+            state.values.append(value)
+            self._prune_events(state.timestamps, state.values, now)
+
+            should_log = self._should_log_locked(now)
+            if should_log:
+                counter_snaps, hist_snaps = self._snapshot_locked(now)
+                snapshot_time = now
+            else:
+                counter_snaps = hist_snaps = []
+                snapshot_time = now
+
+        if should_log and (counter_snaps or hist_snaps):
+            self._log_snapshot(counter_snaps, hist_snaps, snapshot_time)
+
+    def _prune_events(
+        self,
+        timestamps: List[float],
+        values: List[float],
+        now: float,
+    ) -> None:
+        """Prunes events older than the sliding window.
+
+        Args:
+            timestamps: List of event timestamps (ascending).
+            values: List of corresponding values or amounts.
+            now: Current time.
+        """
+        if self.window_seconds is None or not timestamps:
+            return
+        cutoff = now - self.window_seconds
+        idx = 0
+        for i, ts in enumerate(timestamps):
+            if ts >= cutoff:
+                idx = i
+                break
+        else:
+            idx = len(timestamps)
+        if idx > 0:
+            del timestamps[:idx]
+            del values[:idx]
+
+    def _should_log_locked(self, now: float) -> bool:
+        """Determines whether to emit a log snapshot (lock must be held).
+
+        This decision is global: if it returns True, all metrics will be
+        logged based on a snapshot taken at this time.
+
+        Args:
+            now: Current timestamp.
+
+        Returns:
+            True if enough time has elapsed since the last log; False otherwise.
+        """
+        last = self._last_log_time
+        if last is None or now - last >= self.log_interval_seconds:
+            self._last_log_time = now
+            return True
+        return False
+
+    def _snapshot_locked(
+        self,
+        now: float,
+    ) -> Tuple[
+        List[Tuple[str, LabelDict, List[float], List[float]]],
+        List[Tuple[str, LabelDict, List[float], Tuple[float, ...]]],
+    ]:
+        """Creates a snapshot of all metric state (lock must be held).
+
+        Args:
+            now: Current timestamp.
+
+        Returns:
+            A tuple (counter_snapshots, histogram_snapshots) where:
+              - counter_snapshots: list of (metric_name, labels, timestamps, amounts)
+              - histogram_snapshots: list of (metric_name, labels, values, buckets)
+        """
+        counter_snaps: List[Tuple[str, LabelDict, List[float], List[float]]] = []
+        hist_snaps: List[Tuple[str, LabelDict, List[float], Tuple[float, ...]]] = []
+
+        # Prune and snapshot counters.
+        for (name, label_key), state in self._counter_state.items():
+            self._prune_events(state.timestamps, state.amounts, now)
+            if not state.timestamps:
+                continue
+            labels = dict(label_key)
+            counter_snaps.append(
+                (
+                    name,
+                    labels,
+                    list(state.timestamps),
+                    list(state.amounts),
+                )
+            )
+
+        # Prune and snapshot histograms.
+        for (name, label_key), state in self._hist_state.items():
+            self._prune_events(state.timestamps, state.values, now)
+            if not state.values:
+                continue
+            labels = dict(label_key)
+            buckets = self._histograms[name].buckets
+            hist_snaps.append(
+                (
+                    name,
+                    labels,
+                    list(state.values),
+                    buckets,
+                )
+            )
+
+        return counter_snaps, hist_snaps
+
+    def _truncate_labels_for_logging(self, labels: LabelDict) -> LabelDict:
+        """Returns a label dict truncated to the configured group depth.
+
+        Args:
+            labels: Original label dictionary.
+
+        Returns:
+            A new dictionary containing at most `group_level` label pairs,
+            chosen by sorted key order. If group_level is None or < 1, returns
+            a shallow copy of the original labels.
+        """
+        if self.group_level is None or self.group_level < 1:
+            return dict(labels)
+        items = sorted(labels.items())
+        return dict(items[: self.group_level])
+
+    def _log(self, message: str) -> None:
+        """Logs a message via the module logger."""
+        logger.info(message)
+
+    def _log_snapshot(
+        self,
+        counter_snaps: List[Tuple[str, LabelDict, List[float], List[float]]],
+        hist_snaps: List[Tuple[str, LabelDict, List[float], Tuple[float, ...]]],
+        snapshot_time: float,
+    ) -> None:
+        """Logs all metrics from a snapshot.
+
+        Args:
+            counter_snaps: Counter snapshot list.
+            hist_snaps: Histogram snapshot list.
+        """
+        entries: List[str] = []
+        for name, labels, timestamps, amounts in counter_snaps:
+            truncated_labels = self._truncate_labels_for_logging(labels)
+            line = self._log_counter(name, truncated_labels, timestamps, amounts, snapshot_time)
+            if line:
+                entries.append(line)
+
+        for name, labels, values, buckets in hist_snaps:
+            truncated_labels = self._truncate_labels_for_logging(labels)
+            line = self._log_histogram(name, truncated_labels, values, buckets, snapshot_time)
+            if line:
+                entries.append(line)
+
+        if entries:
+            self._log("  ".join(entries))
+
+    def _log_counter(
+        self,
+        name: str,
+        labels: LabelDict,
+        timestamps: List[float],
+        amounts: List[float],
+        snapshot_time: float,
+    ) -> Optional[str]:
+        """Computes counter stats and returns formatted line."""
+        if not timestamps:
+            return None
+
+        total = sum(amounts)
+        window_start = timestamps[0]
+        if self.window_seconds is not None:
+            window_start = max(window_start, snapshot_time - self.window_seconds)
+        min_duration = self.log_interval_seconds if self.log_interval_seconds > 0 else 1e-3
+        duration = max(snapshot_time - window_start, min_duration)
+        rate = total / duration
+
+        label_str = _format_label_string(labels)
+        return f"{name}{label_str}={rate:.2f}/s"
+
+    def _log_histogram(
+        self,
+        name: str,
+        labels: LabelDict,
+        values: List[float],
+        buckets: Tuple[float, ...],
+        snapshot_time: float,
+    ) -> Optional[str]:
+        """Computes histogram stats and returns formatted line."""
+        if not values:
+            return None
+
+        sorted_vals = sorted(values)
+        n = len(sorted_vals)
+
+        def percentile(p: float) -> float:
+            if n == 1:
+                return sorted_vals[0]
+            pos = (p / 100.0) * (n - 1)
+            lo = int(pos)
+            hi = min(lo + 1, n - 1)
+            if lo == hi:
+                return sorted_vals[lo]
+            w = pos - lo
+            return sorted_vals[lo] * (1 - w) + sorted_vals[hi] * w
+
+        p50 = percentile(50.0)
+        p95 = percentile(95.0)
+        p99 = percentile(99.0)
+
+        label_str = _format_label_string(labels)
+        formatted = ",".join([_format_duration(p50), _format_duration(p95), _format_duration(p99)])
+        return f"{name}{label_str}={formatted}"
+
+
+def _format_label_string(labels: LabelDict) -> str:
+    if not labels:
+        return "{}"
+    ordered = ",".join(f"{key}={value}" for key, value in sorted(labels.items()))
+    return f"{{{ordered}}}"
+
+
+def _format_duration(value: float) -> str:
+    abs_value = abs(value)
+    if abs_value >= 1.0:
+        return f"{value:.2f}s"
+    if abs_value >= 1e-3:
+        return f"{value * 1_000:.2f}ms"
+    if abs_value >= 1e-6:
+        return f"{value * 1_000_000:.2f}µs"
+    return f"{value * 1_000_000_000:.2f}ns"
+
+
+class PrometheusMetricsBackend(MetricsBackend):
+    """Metrics backend that forwards events to prometheus_client.
+
+    All metrics must be registered before use. This backend does not compute
+    any aggregations; it only updates Prometheus metrics.
+
+    Thread-safety: Registration is protected by a lock. Metric updates assume metrics
+    are registered during initialization and then remain stable.
+    """
+
+    def __init__(self) -> None:
+        """Initializes PrometheusMetricsBackend.
+
+        Raises:
+            ImportError: If prometheus_client is not installed.
+        """
+        try:
+            import prometheus_client  # type: ignore
+        except ImportError:
+            raise ImportError(
+                "prometheus_client is not installed. Please either install it or use ConsoleMetricsBackend instead."
+            )
+
+        self._counters: Dict[str, _CounterDef] = {}
+        self._histograms: Dict[str, _HistogramDef] = {}
+        self._prom_counters: Dict[str, Any] = {}
+        self._prom_histograms: Dict[str, Any] = {}
+
+        self._lock = threading.Lock()
+
+    def register_counter(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+    ) -> None:
+        """Registers a Prometheus counter metric."""
+        from prometheus_client import Counter as PromCounter
+
+        label_tuple = _normalize_label_names(label_names)
+
+        with self._lock:
+            if name in self._histograms:
+                raise ValueError(f"Metric '{name}' already registered as histogram.")
+
+            existing = self._counters.get(name)
+            if existing is not None:
+                if existing.label_names != label_tuple:
+                    raise ValueError(
+                        f"Counter '{name}' already registered with labels "
+                        f"{existing.label_names}, got {label_tuple}."
+                    )
+                return
+
+            self._counters[name] = _CounterDef(name=name, label_names=label_tuple)
+
+            prom_counter = PromCounter(
+                name,
+                f"Counter {name}",
+                labelnames=label_tuple,
+            )
+            self._prom_counters[name] = prom_counter
+
+    def register_histogram(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+        buckets: Optional[Sequence[float]] = None,
+    ) -> None:
+        """Registers a Prometheus histogram metric."""
+        from prometheus_client import Histogram as PromHistogram
+
+        label_tuple = _normalize_label_names(label_names)
+        bucket_tuple = tuple(buckets) if buckets is not None else ()
+
+        with self._lock:
+            if name in self._counters:
+                raise ValueError(f"Metric '{name}' already registered as counter.")
+
+            existing = self._histograms.get(name)
+            if existing is not None:
+                if existing.label_names != label_tuple or existing.buckets != bucket_tuple:
+                    raise ValueError(
+                        f"Histogram '{name}' already registered with "
+                        f"labels={existing.label_names}, "
+                        f"buckets={existing.buckets}."
+                    )
+                return
+
+            self._histograms[name] = _HistogramDef(
+                name=name,
+                label_names=label_tuple,
+                buckets=bucket_tuple,
+            )
+
+            if bucket_tuple:
+                prom_hist = PromHistogram(
+                    name,
+                    f"Histogram {name}",
+                    labelnames=label_tuple,
+                    buckets=bucket_tuple,
+                )
+            else:
+                prom_hist = PromHistogram(
+                    name,
+                    f"Histogram {name}",
+                    labelnames=label_tuple,
+                )
+
+            self._prom_histograms[name] = prom_hist
+
+    def inc_counter(
+        self,
+        name: str,
+        amount: float = 1.0,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Increments a registered Prometheus counter."""
+        labels = labels or {}
+        definition = self._counters.get(name)
+        if definition is None:
+            raise ValueError(f"Counter '{name}' is not registered.")
+
+        prom_counter = self._prom_counters[name]
+        if definition.label_names:
+            label_key = _validate_labels("counter", name, labels, definition.label_names)
+            prom_counter.labels(**dict(label_key)).inc(amount)
+        else:
+            prom_counter.inc(amount)
+
+    def observe_histogram(
+        self,
+        name: str,
+        value: float,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Records an observation for a registered Prometheus histogram."""
+        labels = labels or {}
+        definition = self._histograms.get(name)
+        if definition is None:
+            raise ValueError(f"Histogram '{name}' is not registered.")
+
+        prom_hist = self._prom_histograms[name]
+        if definition.label_names:
+            label_key = _validate_labels("histogram", name, labels, definition.label_names)
+            prom_hist.labels(**dict(label_key)).observe(value)
+        else:
+            prom_hist.observe(value)
+
+
+class MultiMetricsBackend(MetricsBackend):
+    """Metrics backend that forwards calls to multiple underlying backends."""
+
+    def __init__(self, backends: Sequence[MetricsBackend]) -> None:
+        """Initializes MultiMetricsBackend.
+
+        Args:
+            backends: Sequence of underlying backends.
+
+        Raises:
+            ValueError: If no backends are provided.
+        """
+        if not backends:
+            raise ValueError("MultiMetricsBackend requires at least one backend.")
+        self._backends = list(backends)
+
+    def register_counter(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+    ) -> None:
+        """Registers a counter metric in all underlying backends."""
+        for backend in self._backends:
+            backend.register_counter(name, label_names=label_names)
+
+    def register_histogram(
+        self,
+        name: str,
+        label_names: Optional[Sequence[str]] = None,
+        buckets: Optional[Sequence[float]] = None,
+    ) -> None:
+        """Registers a histogram metric in all underlying backends."""
+        for backend in self._backends:
+            backend.register_histogram(
+                name,
+                label_names=label_names,
+                buckets=buckets,
+            )
+
+    def inc_counter(
+        self,
+        name: str,
+        amount: float = 1.0,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Increments a counter metric in all underlying backends."""
+        for backend in self._backends:
+            backend.inc_counter(name, amount=amount, labels=labels)
+
+    def observe_histogram(
+        self,
+        name: str,
+        value: float,
+        labels: Optional[LabelDict] = None,
+    ) -> None:
+        """Records a histogram observation in all underlying backends."""
+        for backend in self._backends:
+            backend.observe_histogram(name, value=value, labels=labels)
+
+
+_prometheus_multiproc_dir: tempfile.TemporaryDirectory[str] | None = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured."""
+
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s", _prometheus_multiproc_dir.name)
+    else:
+        logger.warning(
+            "Found PROMETHEUS_MULTIPROC_DIR was set by user. " "This directory must be wiped between multiple runs."
+        )
+
+
+def get_prometheus_registry() -> CollectorRegistry:
+    """Get the appropriate prometheus registry based on multiprocessing configuration."""
+    from prometheus_client import REGISTRY, CollectorRegistry, multiprocess
+
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+
+    return REGISTRY
+
+
+def shutdown_metrics():
+    """Shutdown prometheus metrics."""
+
+    from prometheus_client import multiprocess
+
+    path = _prometheus_multiproc_dir
+    if path is None:
+        return
+    try:
+        pid = os.getpid()
+        multiprocess.mark_process_dead(pid, path.name)  # type: ignore
+        logger.debug("Marked Prometheus metrics for process %d as dead", pid)
+    except Exception as e:
+        logger.error("Error during metrics cleanup: %s", str(e))

agentlightning/utils/otel.py

@@ -0,0 +1,401 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Utilities shared for OpenTelemetry span (attributes) support."""
+
+import logging
+from typing import Any, Dict, List, Sequence, Union, cast
+from warnings import filterwarnings
+
+import opentelemetry.trace as trace_api
+from agentops.sdk.exporters import OTLPSpanExporter
+from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, SynchronousMultiSpanProcessor, Tracer
+from opentelemetry.sdk.trace import TracerProvider as TracerProviderImpl
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor
+from opentelemetry.sdk.util.instrumentation import InstrumentationInfo, InstrumentationScope
+from opentelemetry.trace import get_tracer_provider as otel_get_tracer_provider
+from pydantic import TypeAdapter
+
+from agentlightning.env_var import LightningEnvVar, resolve_bool_env_var
+from agentlightning.semconv import LightningSpanAttributes, LinkAttributes, LinkPydanticModel
+from agentlightning.types import SpanLike
+from agentlightning.utils.otlp import LightningStoreOTLPExporter
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "full_qualified_name",
+    "get_tracer_provider",
+    "get_tracer",
+    "make_tag_attributes",
+    "extract_tags_from_attributes",
+    "make_link_attributes",
+    "query_linked_spans",
+    "extract_links_from_attributes",
+    "filter_attributes",
+    "filter_and_unflatten_attributes",
+    "flatten_attributes",
+    "unflatten_attributes",
+]
+
+
+def full_qualified_name(obj: type) -> str:
+    if str(obj.__module__) == "builtins":
+        return obj.__qualname__
+    return f"{obj.__module__}.{obj.__qualname__}"
+
+
+def get_tracer_provider(inspect: bool = True) -> TracerProviderImpl:
+    """Get the OpenTelemetry tracer provider configured for Agent Lightning.
+
+    Args:
+        inspect: Whether to inspect the tracer provider and log its configuration.
+            When it's on, make sure you also set the logger level to DEBUG to see the logs.
+    """
+    from agentlightning.tracer.otel import LightningSpanProcessor
+
+    if hasattr(trace_api, "_TRACER_PROVIDER") and trace_api._TRACER_PROVIDER is None:  # type: ignore[attr-defined]
+        raise RuntimeError("Tracer is not initialized. Cannot emit a meaningful span.")
+    tracer_provider = otel_get_tracer_provider()
+    if not isinstance(tracer_provider, TracerProviderImpl):
+        logger.error(
+            "Tracer provider is expected to be an instance of opentelemetry.sdk.trace.TracerProvider, found: %s",
+            full_qualified_name(type(tracer_provider)),
+        )
+        return cast(TracerProviderImpl, tracer_provider)
+
+    if not inspect:
+        return tracer_provider
+
+    emitter_debug = resolve_bool_env_var(LightningEnvVar.AGL_EMITTER_DEBUG, fallback=None)
+    logger_effective_level = logger.getEffectiveLevel()
+    if emitter_debug is True and logger_effective_level > logging.DEBUG:
+        logger.warning(
+            "Emitter debug logging is enabled but logging level is not set to DEBUG. Nothing will be logged."
+        )
+
+    if emitter_debug is None:
+        # Set to true by default if the logging level is lower than DEBUG
+        emitter_debug = logging.DEBUG >= logger_effective_level
+
+    if emitter_debug:
+        active_span_processor = tracer_provider._active_span_processor  # pyright: ignore[reportPrivateUsage]
+        processors: List[str] = []
+        active_span_processor_cls = active_span_processor.__class__.__name__
+        for processor in active_span_processor._span_processors:  # pyright: ignore[reportPrivateUsage]
+            if isinstance(processor, LightningSpanProcessor):
+                # The legacy case for tracers without OTLP support.
+                processors.append(f"{active_span_processor_cls} - {processor!r}")
+            elif isinstance(processor, (SimpleSpanProcessor, BatchSpanProcessor)):
+                processor_cls = processor.__class__.__name__
+                if isinstance(processor.span_exporter, LightningStoreOTLPExporter):
+                    # This should be the main path now.
+                    processors.append(f"{active_span_processor_cls} - {processor_cls} - {processor.span_exporter!r}")
+                elif isinstance(processor.span_exporter, OTLPSpanExporter):
+                    # You need to be careful if the code goes into this path.
+                    endpoint = processor.span_exporter._endpoint  # pyright: ignore[reportPrivateUsage]
+                    processors.append(
+                        f"{active_span_processor_cls} - {processor_cls} - "
+                        f"{processor.span_exporter.__class__.__name__}(endpoint={endpoint!r})"
+                    )
+                else:
+                    # Other cases like Console Span Exporter.
+                    processors.append(
+                        f"{active_span_processor_cls} - {processor_cls} - {processor.span_exporter.__class__.__name__}"
+                    )
+            else:
+                processors.append(f"{active_span_processor_cls} - {processor.__class__.__name__}")
+
+        logger.debug(f"Tracer provider: {tracer_provider!r}. Active span processors:")
+        for processor in processors:
+            logger.debug("  * " + processor)
+
+    return tracer_provider
+
+
+def get_tracer(use_active_span_processor: bool = True) -> trace_api.Tracer:
+    """Resolve the OpenTelemetry tracer configured for Agent Lightning.
+
+    Args:
+        use_active_span_processor: Whether to use the active span processor.
+
+    Returns:
+        OpenTelemetry tracer tagged with the `agentlightning` instrumentation name.
+
+    Raises:
+        RuntimeError: If OpenTelemetry was not initialized before calling this helper.
+    """
+    if hasattr(trace_api, "_TRACER_PROVIDER") and trace_api._TRACER_PROVIDER is None:  # type: ignore[attr-defined]
+        raise RuntimeError("Tracer is not initialized. Cannot emit a meaningful span.")
+
+    tracer_provider = get_tracer_provider(inspect=True)  # inspection is on by default
+
+    if use_active_span_processor:
+        return tracer_provider.get_tracer("agentlightning")
+
+    else:
+        filterwarnings(
+            "ignore",
+            message=r"You should use InstrumentationScope. Deprecated since version 1.11.1.",
+            category=DeprecationWarning,
+            module="opentelemetry.sdk.trace",
+        )
+
+        return Tracer(
+            tracer_provider.sampler,
+            tracer_provider.resource,
+            # We use an empty span processor to avoid emitting spans to the tracer
+            SynchronousMultiSpanProcessor(),
+            tracer_provider.id_generator,
+            InstrumentationInfo("agentlightning", "", ""),  # type: ignore
+            SpanLimits(),
+            InstrumentationScope(
+                "agentlightning",
+                "",
+                "",
+                {},
+            ),
+        )
+
+
+def make_tag_attributes(tags: List[str]) -> Dict[str, Any]:
+    """Convert a list of tags into flattened attributes for span tagging.
+
+    There is no syntax enforced for tags, they are just strings. For example:
+
+    ```python
+    ["gen_ai.model:gpt-4", "reward.extrinsic"]
+    ```
+    """
+    return flatten_attributes({LightningSpanAttributes.TAG.value: tags})
+
+
+def extract_tags_from_attributes(attributes: Dict[str, Any]) -> List[str]:
+    """Extract tag attributes from flattened span attributes.
+
+    Args:
+        attributes: A dictionary of flattened span attributes.
+    """
+    maybe_tag_list = filter_and_unflatten_attributes(attributes, LightningSpanAttributes.TAG.value)
+    return TypeAdapter(List[str]).validate_python(maybe_tag_list)
+
+
+def make_link_attributes(links: Dict[str, str]) -> Dict[str, Any]:
+    """Convert a dictionary of links into flattened attributes for span linking.
+
+    Links example:
+
+    ```python
+    {
+        "gen_ai.response.id": "response-123",
+        "span_id": "abcd-efgh-ijkl",
+    }
+    ```
+    """
+    link_list: List[Dict[str, str]] = []
+    for key, value in links.items():
+        if not isinstance(value, str):  # pyright: ignore[reportUnnecessaryIsInstance]
+            raise ValueError(f"Link value must be a string, got {type(value)} for key '{key}'")
+        link_list.append({LinkAttributes.KEY_MATCH.value: key, LinkAttributes.VALUE_MATCH.value: value})
+    return flatten_attributes({LightningSpanAttributes.LINK.value: link_list})
+
+
+def query_linked_spans(spans: Sequence[SpanLike], links: List[LinkPydanticModel]) -> List[SpanLike]:
+    """Query spans that are linked by the given link attributes.
+
+    Args:
+        spans: A sequence of spans to search.
+        links: A list of link attributes to match.
+
+    Returns:
+        A list of spans that match the given link attributes.
+    """
+    matched_spans: List[SpanLike] = []
+
+    for span in spans:
+        span_attributes = span.attributes or {}
+        is_match = True
+        for link in links:
+            # trace_id and span_id must be full match.
+            if link.key_match == "trace_id":
+                if isinstance(span, ReadableSpan):
+                    trace_id = trace_api.format_trace_id(span.context.trace_id) if span.context else None
+                else:
+                    trace_id = span.trace_id
+                if trace_id != link.value_match:
+                    is_match = False
+                    break
+
+            elif link.key_match == "span_id":
+                if isinstance(span, ReadableSpan):
+                    span_id = trace_api.format_span_id(span.context.span_id) if span.context else None
+                else:
+                    span_id = span.span_id
+                if span_id != link.value_match:
+                    is_match = False
+                    break
+
+            else:
+                attribute = span_attributes.get(link.key_match)
+                # attributes must also be a full match currently.
+                if attribute != link.value_match:
+                    is_match = False
+                    break
+
+        if is_match:
+            matched_spans.append(span)
+
+    return matched_spans
+
+
+def extract_links_from_attributes(attributes: Dict[str, Any]) -> List[LinkPydanticModel]:
+    """Extract link attributes from flattened span attributes.
+
+    Args:
+        attributes: A dictionary of flattened span attributes.
+    """
+    maybe_link_list = filter_and_unflatten_attributes(attributes, LightningSpanAttributes.LINK.value)
+    return TypeAdapter(List[LinkPydanticModel]).validate_python(maybe_link_list)
+
+
+def filter_attributes(attributes: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+    """Filter attributes that start with the given prefix.
+
+    The attribute must start with `prefix.` or be exactly `prefix` to be included.
+
+    Args:
+        attributes: A dictionary of span attributes.
+        prefix: The prefix to filter by.
+
+    Returns:
+        A dictionary of attributes that start with the given prefix.
+    """
+    return {k: v for k, v in attributes.items() if k.startswith(prefix + ".") or k == prefix}
+
+
+def filter_and_unflatten_attributes(attributes: Dict[str, Any], prefix: str) -> Union[Dict[str, Any], List[Any]]:
+    """Filter attributes that start with the given prefix and unflatten them.
+    The prefix will be removed during unflattening.
+
+    Args:
+        attributes: A dictionary of span attributes.
+        prefix: The prefix to filter by.
+
+    Returns:
+        A nested dictionary or list of attributes that start with the given prefix.
+    """
+    filtered_attributes = filter_attributes(attributes, prefix)
+    stripped_attributes: Dict[str, Any] = {}
+    for k, v in filtered_attributes.items():
+        if k == prefix:
+            raise ValueError(f"Cannot unflatten attribute with key exactly equal to prefix: {prefix}")
+        else:
+            stripped_key = k[len(prefix) + 1 :]  # +1 to remove the dot
+        stripped_attributes[stripped_key] = v
+    return unflatten_attributes(stripped_attributes)
+
+
+def flatten_attributes(nested_data: Union[Dict[str, Any], List[Any]]) -> Dict[str, Any]:
+    """Flatten a nested dictionary or list into a flat dictionary with dotted keys.
+
+    This function recursively traverses dictionaries and lists, producing a flat
+    key-value mapping where nested paths are represented via dot-separated keys.
+    Lists are indexed numerically.
+
+    Example:
+
+        >>> flatten_attributes({"a": {"b": 1, "c": [2, 3]}})
+        {"a.b": 1, "a.c.0": 2, "a.c.1": 3}
+
+    Args:
+        nested_data: A nested structure composed of dictionaries, lists, or
+            primitive values.
+
+    Returns:
+        A flat dictionary mapping dotted-string paths to primitive values.
+    """
+
+    flat: Dict[str, Any] = {}
+
+    def _walk(value: Any, prefix: str = "") -> None:
+        if isinstance(value, dict):
+            for k, v in cast(Dict[Any, Any], value).items():
+                if not isinstance(k, str):
+                    raise ValueError(
+                        f"Only string keys are supported in dictionaries, got '{k}' of type {type(k)} in {prefix}"
+                    )
+                new_prefix = f"{prefix}.{k}" if prefix else k
+                _walk(v, new_prefix)
+        elif isinstance(value, list):
+            for idx, item in enumerate(cast(List[Any], value)):
+                new_prefix = f"{prefix}.{idx}" if prefix else str(idx)
+                _walk(item, new_prefix)
+        else:
+            flat[prefix] = value
+
+    _walk(nested_data)
+    return flat
+
+
+def unflatten_attributes(flat_data: Dict[str, Any]) -> Union[Dict[str, Any], List[Any]]:
+    """Reconstruct a nested dictionary/list structure from a flat dictionary.
+
+    Keys are dot-separated paths. Segments that are digit strings will only
+    become list indices if *all* keys in that dict form a consecutive
+    0..n-1 range. Otherwise they remain dict keys.
+
+    Example:
+
+        >>> unflatten_attributes({"a.b": 1, "a.c.0": 2, "a.c.1": 3})
+        {"a": {"b": 1, "c": [2, 3]}}
+
+    Args:
+        flat_data: A dictionary whose keys are dot-separated paths and whose
+            values are primitive data elements.
+
+    Returns:
+        A nested dictionary (and lists where appropriate) corresponding to
+        the flattened structure.
+    """
+    # 1) Build a pure dict tree first (no lists yet)
+    root: Dict[str, Any] = {}
+
+    for flat_key, value in flat_data.items():
+        parts = flat_key.split(".")
+        curr: Dict[str, Any] = root
+
+        for part in parts[:-1]:
+            # Ensure intermediate node is a dict
+            if part not in curr or not isinstance(curr[part], dict):
+                curr[part] = {}
+            curr = curr[part]  # type: ignore[assignment]
+
+        curr[parts[-1]] = value
+
+    # 2) Recursively convert dicts-with-consecutive-numeric-keys into lists
+    def convert(node: Union[Dict[str, Any], List[Any]]) -> Union[Dict[str, Any], List[Any]]:
+        if isinstance(node, dict):
+            # First convert children
+            for k, v in list(node.items()):
+                node[k] = convert(v)
+
+            if not node:
+                # empty dict stays dict
+                return node
+
+            # Check if keys are all numeric strings
+            keys = list(node.keys())
+            if all(isinstance(k, str) and k.isdigit() for k in keys):  # pyright: ignore[reportUnnecessaryIsInstance]
+                indices = sorted(int(k) for k in keys)
+                # Must be exactly 0..n-1
+                if indices == list(range(len(indices))):
+                    return [node[str(i)] for i in range(len(indices))]
+
+            return node
+
+        if isinstance(node, list):  # pyright: ignore[reportUnnecessaryIsInstance]
+            return [convert(v) for v in node]
+
+        # Keep as is
+        return node
+
+    return convert(root)

agentlightning/utils/otlp.py

@@ -0,0 +1,474 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from __future__ import annotations
+
+import gzip
+import logging
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar
+
+from fastapi import Request, Response
+from google.protobuf import json_format
+from google.rpc.status_pb2 import Status
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import (
+    ExportLogsServiceRequest,
+    ExportLogsServiceResponse,
+)
+from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (
+    ExportMetricsServiceRequest,
+    ExportMetricsServiceResponse,
+)
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+    ExportTraceServiceRequest,
+    ExportTraceServiceResponse,
+)
+from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
+from opentelemetry.proto.resource.v1.resource_pb2 import Resource as ProtoResource
+from opentelemetry.proto.trace.v1.trace_pb2 import Span as ProtoSpan
+from opentelemetry.proto.trace.v1.trace_pb2 import Status as ProtoStatus
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExportResult
+from opentelemetry.util.types import AttributeValue
+
+from agentlightning.semconv import LightningResourceAttributes
+from agentlightning.types.tracer import (
+    Attributes,
+    Event,
+    Link,
+    OtelResource,
+    Span,
+    SpanContext,
+    TraceStatus,
+    convert_timestamp,
+)
+
+PROTOBUF_CT = "application/x-protobuf"
+
+logger = logging.getLogger(__name__)
+
+
+T_request = TypeVar("T_request", ExportLogsServiceRequest, ExportMetricsServiceRequest, ExportTraceServiceRequest)
+T_response = TypeVar("T_response", ExportLogsServiceResponse, ExportMetricsServiceResponse, ExportTraceServiceResponse)
+
+
+async def handle_otlp_export(
+    request: Request,
+    request_message_cls: Type[T_request],
+    response_message_cls: Type[T_response],
+    message_callback: Optional[Callable[[T_request], Awaitable[None]]],
+    signal_name: str,
+) -> Response:
+    """
+    Generic handler for /v1/traces, /v1/metrics, /v1/logs.
+
+    Convert the OTLP Protobuf request to a JSON-like object.
+    """
+    content_type = request.headers.get("Content-Type", "").split(";")[0].strip()
+
+    if content_type != PROTOBUF_CT:
+        # For brevity we only support binary protobuf here.
+        return _bad_request_response(
+            request,
+            f"Unsupported Content-Type '{content_type}', expected '{PROTOBUF_CT}'",
+            content_type=PROTOBUF_CT,
+        )
+
+    raw_body = await request.body()
+    body = _read_body_maybe_gzip(request, raw_body)
+
+    # Empty request is allowed and should still succeed.
+    if not body:
+        req_msg = request_message_cls()
+    else:
+        req_msg = request_message_cls()
+        try:
+            req_msg.ParseFromString(body)
+        except Exception as exc:
+            return _bad_request_response(request, f"Unable to parse OTLP {signal_name} payload: {exc}")
+
+    if message_callback is not None:
+        await message_callback(req_msg)
+
+    # Build success response. Partial success field is left unset.
+    resp_msg = response_message_cls()
+
+    # Encode response in the same Content-Type as request.
+    if content_type == PROTOBUF_CT:
+        resp_bytes = resp_msg.SerializeToString()
+    else:
+        resp_bytes = json_format.MessageToJson(resp_msg).encode("utf-8")
+
+    resp_bytes, headers = _maybe_gzip_response(request, resp_bytes)
+
+    return Response(
+        content=resp_bytes,
+        media_type=content_type,
+        status_code=200,
+        headers=headers,
+    )
+
+
+async def spans_from_proto(
+    request: ExportTraceServiceRequest,
+    sequence_id_bulk_issuer: Callable[[Sequence[Tuple[str, str]]], Awaitable[Sequence[int]]],
+) -> List[Span]:
+    """Parse an OTLP proto payload into List[Span].
+
+    A store is needed here for generating a sequence ID for each span.
+    """
+    output_spans: List[Span] = []
+
+    for resource_spans in request.resource_spans:
+        # Resource-level attributes & IDs
+        resource_attrs = _kv_list_to_dict(resource_spans.resource.attributes)
+        # rollout_id, attempt_id from resource attributes when present.
+        rollout_id_resource = resource_attrs.get(LightningResourceAttributes.ROLLOUT_ID.value)
+        attempt_id_resource = resource_attrs.get(LightningResourceAttributes.ATTEMPT_ID.value)
+        # If sequence id is provided, all the spans will share the same sequence ID.
+        # unless otherwise overridden by span-level attributes.
+        sequence_id_resource = resource_attrs.get(LightningResourceAttributes.SPAN_SEQUENCE_ID.value)
+
+        otel_resource = _resource_from_proto(resource_spans.resource, getattr(resource_spans, "schema_url", ""))
+
+        # Each ScopeSpans contains multiple spans
+        for scope_spans in resource_spans.scope_spans:
+            for proto_span in scope_spans.spans:
+                trace_id_hex = _bytes_to_trace_id_hex(proto_span.trace_id)
+                span_id_hex = _bytes_to_span_id_hex(proto_span.span_id)
+                parent_id_hex = _bytes_to_span_id_hex(proto_span.parent_span_id) if proto_span.parent_span_id else None
+
+                # Status
+                status_code_str = _STATUS_CODE_MAP.get(proto_span.status.code, "UNSET")
+                status = TraceStatus(
+                    status_code=status_code_str,
+                    description=proto_span.status.message or None,
+                )
+
+                # Attributes
+                span_attrs = _kv_list_to_dict(proto_span.attributes)
+
+                # Context
+                context = SpanContext(
+                    trace_id=trace_id_hex,
+                    span_id=span_id_hex,
+                    is_remote=False,
+                    trace_state={},
+                )
+
+                # Try to get if span attributes contain something like rollout_id or attempt_id
+                # Override the resource-level attributes with the span-level attributes if present.
+                rollout_id_span = span_attrs.get(LightningResourceAttributes.ROLLOUT_ID.value)
+                attempt_id_span = span_attrs.get(LightningResourceAttributes.ATTEMPT_ID.value)
+                sequence_id_span = span_attrs.get(LightningResourceAttributes.SPAN_SEQUENCE_ID.value)
+
+                # Normalize to regular strings and ints
+                rollout_id_raw = rollout_id_span if rollout_id_span is not None else rollout_id_resource
+                attempt_id_raw = attempt_id_span if attempt_id_span is not None else attempt_id_resource
+                sequence_id_raw = sequence_id_span if sequence_id_span is not None else sequence_id_resource
+
+                rollout_id, attempt_id = _normalize_rollout_attempt_id(rollout_id_raw, attempt_id_raw)
+                sequence_id = _normalize_sequence_id(sequence_id_raw)
+
+                if rollout_id is None or attempt_id is None:
+                    logger.warning(
+                        "Both rollout_id and attempt_id must be present in resource attributes. "
+                        "Spans will not be able to log to the store because of missing IDs: rollout_id=%s, attempt_id=%s, sequence_id=%s",
+                        rollout_id,
+                        attempt_id,
+                        sequence_id,
+                    )
+                    continue
+
+                # Generate a new sequence ID if not provided
+                if sequence_id is None:
+                    current_sequence_id = -1
+                elif sequence_id < 0:
+                    logger.error(
+                        "Invalid sequence_id value in resource attributes: %r. Must be a positive integer. Regenerating one.",
+                        sequence_id,
+                    )
+                    current_sequence_id = -1
+                else:
+                    current_sequence_id = sequence_id
+
+                # Build Span
+                span = Span(
+                    rollout_id=rollout_id,
+                    attempt_id=attempt_id,
+                    sequence_id=current_sequence_id,
+                    trace_id=trace_id_hex,
+                    span_id=span_id_hex,
+                    parent_id=parent_id_hex,
+                    name=proto_span.name,
+                    status=status,
+                    attributes=span_attrs,
+                    events=_events_from_proto(proto_span),
+                    links=_links_from_proto(proto_span),
+                    start_time=convert_timestamp(proto_span.start_time_unix_nano),
+                    end_time=convert_timestamp(proto_span.end_time_unix_nano),
+                    context=context,
+                    parent=None,  # OTLP only has parent_span_id; we don't have full SpanContext
+                    resource=otel_resource,
+                )
+
+                output_spans.append(span)
+
+    # Finalize the sequence IDs
+    bulk_issue_requests = [(span.rollout_id, span.attempt_id) for span in output_spans if span.sequence_id < 0]
+    bulk_sequence_ids = await sequence_id_bulk_issuer(bulk_issue_requests)
+    for span, sequence_id in zip(
+        [span for span in output_spans if span.sequence_id < 0], bulk_sequence_ids, strict=True
+    ):
+        span.sequence_id = sequence_id
+
+    return output_spans
+
+
+class LightningStoreOTLPExporter(OTLPSpanExporter):
+    """OTLP Exporter that write to a LightningStore-compatible backend.
+
+    The backend requires two special attributes on each span:
+
+    - `agentlightning.rollout_id`: The rollout ID to associate the span with.
+    - `agentlightning.attempt_id`: The attempt ID to associate the span with.
+
+    It can optionally use the following attribute to sequence spans:
+
+    - `agentlightning.span_sequence_id`: A decimal string representing the sequence ID of the span.
+    """
+
+    _default_endpoint: Optional[str] = None
+    _rollout_id: Optional[str] = None
+    _attempt_id: Optional[str] = None
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}("
+            + f"endpoint={self.endpoint!r}, "
+            + f"rollout_id={self.rollout_id!r}, "
+            + f"attempt_id={self.attempt_id!r}, "
+            + f"should_bypass={self.should_bypass()!r})"
+        )
+
+    @property
+    def endpoint(self) -> Optional[str]:
+        """The endpoint to submit the spans to."""
+        if hasattr(self, "_endpoint"):
+            return self._endpoint
+        return None
+
+    @property
+    def rollout_id(self) -> Optional[str]:
+        """The rollout ID to submit the spans to."""
+        if hasattr(self, "_rollout_id"):
+            return self._rollout_id
+        return None
+
+    @property
+    def attempt_id(self) -> Optional[str]:
+        """The attempt ID to submit the spans to."""
+        if hasattr(self, "_attempt_id"):
+            return self._attempt_id
+        return None
+
+    def enable_store_otlp(self, endpoint: str, rollout_id: str, attempt_id: str) -> None:
+        """Enable storing OTLP data to a specific LightningStore rollout/attempt."""
+        self._rollout_id = rollout_id
+        self._attempt_id = attempt_id
+
+        self._default_endpoint = self._endpoint
+        self._endpoint = endpoint
+
+    def disable_store_otlp(self) -> None:
+        """Disable storing OTLP data to LightningStore."""
+        self._rollout_id = None
+        self._attempt_id = None
+        if self._default_endpoint is not None:
+            self._endpoint = self._default_endpoint
+
+    def should_bypass(self) -> bool:
+        """Check if the exporter should bypass the default export if rollout_id and attempt_id are not set."""
+        return True
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        if self._rollout_id is not None and self._attempt_id is not None:
+            # rollout_id and attempt_id are present in resource attributes
+            # It means that the server supports OTLP endpoint.
+            for span in spans:
+                # Override the resources so that the server knows where the request comes from.
+                span._resource = span._resource.merge(  # pyright: ignore[reportPrivateUsage]
+                    Resource.create(
+                        {
+                            LightningResourceAttributes.ROLLOUT_ID.value: self._rollout_id,
+                            LightningResourceAttributes.ATTEMPT_ID.value: self._attempt_id,
+                        }
+                    )
+                )
+            return super().export(spans)
+        elif not self.should_bypass():
+            logger.debug("Rollout ID and Attempt ID not set; using default OTLP exporter behavior.")
+            return super().export(spans)
+        else:
+            logger.debug("Rollout ID and Attempt ID not set; bypassing export.")
+            return SpanExportResult.SUCCESS
+
+
+def _read_body_maybe_gzip(request: Request, raw_body: bytes) -> bytes:
+    """
+    Decompress body if Content-Encoding: gzip; otherwise return as is.
+    """
+    encoding = request.headers.get("Content-Encoding", "").lower()
+    if encoding == "gzip":
+        return gzip.decompress(raw_body)
+    return raw_body
+
+
+def _maybe_gzip_response(request: Request, payload: bytes) -> Tuple[bytes, Dict[str, str]]:
+    """
+    If Accept-Encoding includes gzip, gzip the payload and set Content-Encoding header.
+    """
+    ae = request.headers.get("Accept-Encoding", "")
+    tokens = [token.split(";")[0].strip().lower() for token in ae.split(",") if token.strip()]
+    headers: Dict[str, str] = {}
+    if "gzip" in tokens:
+        payload = gzip.compress(payload)
+        headers["Content-Encoding"] = "gzip"
+    return payload, headers
+
+
+def _bad_request_response(request: Request, message: str, content_type: str = PROTOBUF_CT) -> Response:
+    """
+    Build a 400 response whose body is a protobuf Status message, encoded
+    in the same Content-Type as the request (OTLP/HTTP requirement).
+    """
+    status_msg = Status(message=message)
+
+    if content_type == PROTOBUF_CT:
+        body = status_msg.SerializeToString()
+    else:
+        # Fallback: JSON representation of Status.
+        body = json_format.MessageToJson(status_msg).encode("utf-8")
+
+    body, headers = _maybe_gzip_response(request, body)
+
+    return Response(
+        content=body,
+        status_code=400,
+        media_type=content_type,
+        headers=headers,
+    )
+
+
+def _normalize_rollout_attempt_id(
+    rollout_id: Optional[AttributeValue], attempt_id: Optional[AttributeValue]
+) -> Tuple[Optional[str], Optional[str]]:
+    """Normalize a rollout or attempt ID to a string."""
+    rollout_id_str = str(rollout_id) if rollout_id is not None else None
+    attempt_id_str = str(attempt_id) if attempt_id is not None else None
+    return rollout_id_str, attempt_id_str
+
+
+def _normalize_sequence_id(sequence_id: Optional[AttributeValue]) -> Optional[int]:
+    """Normalize a sequence ID to an integer."""
+    if sequence_id is None:
+        return None
+    try:
+        sequence_id_int = int(str(sequence_id))
+    except (ValueError, TypeError):
+        logger.warning(
+            "Invalid sequence_id value in resource attributes: %r. Must be an integer or string representing an integer. Assuming None.",
+            sequence_id,
+        )
+        sequence_id_int = None
+    return sequence_id_int
+
+
+def _any_value_to_python(value: AnyValue) -> Any:
+    """Convert OTLP AnyValue -> plain Python value."""
+    kind = value.WhichOneof("value")
+    if kind is None:
+        return None
+    if kind == "string_value":
+        return value.string_value
+    if kind == "bool_value":
+        return value.bool_value
+    if kind == "int_value":
+        return int(value.int_value)
+    if kind == "double_value":
+        return float(value.double_value)
+    if kind == "array_value":
+        return [_any_value_to_python(v) for v in value.array_value.values]
+    if kind == "kvlist_value":
+        # Map<string, AnyValue> -> dict
+        return {kv.key: _any_value_to_python(kv.value) for kv in value.kvlist_value.values}
+    if kind == "bytes_value":
+        # Serialize bytes as hex string to stay JSON-friendly
+        return value.bytes_value.hex()
+    return None
+
+
+def _kv_list_to_dict(kvs: Sequence[KeyValue]) -> Attributes:
+    """Convert repeated KeyValue -> Attributes dict."""
+    return {kv.key: _any_value_to_python(kv.value) for kv in kvs}
+
+
+_STATUS_CODE_MAP = {
+    ProtoStatus.STATUS_CODE_UNSET: "UNSET",
+    ProtoStatus.STATUS_CODE_OK: "OK",
+    ProtoStatus.STATUS_CODE_ERROR: "ERROR",
+}
+
+
+def _bytes_to_trace_id_hex(b: bytes) -> str:
+    # OTLP uses 16-byte trace IDs; format as 32-char hex
+    if not b:
+        return "0" * 32
+    return b.hex().rjust(32, "0")
+
+
+def _bytes_to_span_id_hex(b: bytes) -> str:
+    # OTLP uses 8-byte span IDs; format as 16-char hex
+    if not b:
+        return "0" * 16
+    return b.hex().rjust(16, "0")
+
+
+def _events_from_proto(span: ProtoSpan) -> List[Event]:
+    """Event converter from OTLP ProtoSpan to List[Event]."""
+    return [
+        Event(
+            name=e.name,
+            attributes=_kv_list_to_dict(e.attributes),
+            timestamp=convert_timestamp(e.time_unix_nano),
+        )
+        for e in span.events
+    ]
+
+
+def _links_from_proto(span: ProtoSpan) -> List[Link]:
+    """Link converter from OTLP ProtoSpan to List[Link]."""
+    links: List[Link] = []
+    for link in span.links:
+        trace_id_hex = _bytes_to_trace_id_hex(link.trace_id)
+        span_id_hex = _bytes_to_span_id_hex(link.span_id)
+        ctx = SpanContext(
+            trace_id=trace_id_hex,
+            span_id=span_id_hex,
+            is_remote=False,
+            trace_state={},  # OTLP trace_state is currently a string; you can parse if needed
+        )
+        links.append(
+            Link(
+                context=ctx,
+                attributes=_kv_list_to_dict(link.attributes) or None,
+            )
+        )
+    return links
+
+
+def _resource_from_proto(resource: ProtoResource, schema_url: str = "") -> OtelResource:
+    return OtelResource(
+        attributes=_kv_list_to_dict(resource.attributes),
+        schema_url=schema_url or "",
+    )

agentlightning/utils/server_launcher.py

@@ -6,6 +6,7 @@ import asyncio
 import inspect
 import logging
 import multiprocessing
+import os
 import queue
 import signal
 import socket
@@ -15,7 +16,7 @@ import traceback
 from contextlib import asynccontextmanager, suppress
 from dataclasses import dataclass
 from multiprocessing.process import BaseProcess
-from typing import Any, AsyncContextManager, AsyncIterator, Dict, Literal, Optional
+from typing import Any, AsyncContextManager, AsyncIterator, Dict, Literal, Optional, cast
 
 import aiohttp
 import requests
@@ -53,6 +54,8 @@ class PythonServerLauncherArgs:
     """
     log_level: int = logging.INFO
     """The log level to use."""
+    access_log: bool = False
+    """Whether to turn on access logs."""
     startup_timeout: float = 60.0
     """The timeout to wait for the server to start up."""
     kill_unhealthy_server: bool = True
@@ -63,6 +66,8 @@ class PythonServerLauncherArgs:
     """The timeout to wait for the thread to join."""
     process_join_timeout: float = 10.0
     """The timeout to wait for the process to join."""
+    timeout_keep_alive: int = 30
+    """The timeout to keep the connection alive."""
 
 
 @dataclass
@@ -156,7 +161,9 @@ async def run_uvicorn_asyncio(
 
         if not uvicorn_server.started:
             # Normally, the program will not reach this point, as the server will throw the exception itself earlier.
-            raise RuntimeError(f"Server did not start up within {timeout:.2f} seconds.") from server_start_exception
+            raise RuntimeError(
+                f"Server did not start up within {time.time() - start_time:.2f} seconds."
+            ) from server_start_exception
 
         logger.info(f"Server started up in {time.time() - start_time:.2f} seconds.")
 
@@ -608,6 +615,13 @@ class PythonServerLauncher:
         self._host: Optional[str] = self.args.host
         self._port: Optional[int] = self.args.port
         self._access_host: Optional[str] = self.args.access_host
+        self.initialize()
+
+    def initialize(self):
+        # ensure the host/port/access_host are set
+        self._ensure_host()
+        self._ensure_port()
+        self._ensure_access_host()
 
         # uvicorn (in-proc asyncio)
         self._uvicorn_server: Optional[uvicorn.Server] = None
@@ -626,6 +640,26 @@ class PythonServerLauncher:
         # is_running flag
         self._is_running: bool = False
 
+    def __getstate__(self):
+        """Control pickling to prevent server state from being sent to subprocesses."""
+        return {
+            "app": self.app,
+            "args": self.args,
+            "serve_context": self.serve_context,
+            "_host": self._host,
+            "_port": self._port,
+            "_access_host": self._access_host,
+        }
+
+    def __setstate__(self, state: Dict[str, Any]):
+        self.app = state["app"]
+        self.args = cast(PythonServerLauncherArgs, state["args"])
+        self.serve_context = state["serve_context"]
+        self._host = state["_host"]
+        self._port = state["_port"]
+        self._access_host = state["_access_host"]
+        self.initialize()
+
     @property
     def endpoint(self) -> str:
         """Return the externally advertised host:port pair regardless of accessibility."""
@@ -744,17 +778,18 @@ class PythonServerLauncher:
         return self._port
 
     def _ensure_access_host(self) -> str:
-        if self.args.access_host is None:
-            if self._ensure_host() in ("0.0.0.0", "::"):
-                # Probe host normalization for 0.0.0.0
-                logger.warning("No access host provided, using default outbound IPv4 address for this machine.")
-                self._access_host = _get_default_ipv4_address()
+        if self._access_host is None:
+            if self.args.access_host is None:
+                if self._ensure_host() in ("0.0.0.0", "::"):
+                    # Probe host normalization for 0.0.0.0
+                    logger.warning("No access host provided, using default outbound IPv4 address for this machine.")
+                    self._access_host = _get_default_ipv4_address()
+                else:
+                    logger.warning("No access host provided, using the host provided.")
+                    self._access_host = self._ensure_host()
             else:
-                logger.warning("No access host provided, using the host provided.")
-                self._access_host = self._ensure_host()
-        else:
-            self._access_host = self.args.access_host
-        return self._access_host
+                self._access_host = self.args.access_host
+        return self._access_host  # type: ignore
 
     def _create_uvicorn_server(self) -> uvicorn.Server:
         config = uvicorn.Config(
@@ -762,7 +797,9 @@ class PythonServerLauncher:
             host=self._ensure_host(),
             port=self._ensure_port(),
             log_level=self.args.log_level,
+            access_log=self.args.access_log,
             loop="asyncio",
+            timeout_keep_alive=self.args.timeout_keep_alive,
         )
         return uvicorn.Server(config)
 
@@ -834,17 +871,19 @@ class PythonServerLauncher:
             evt: ChildEvent = await asyncio.to_thread(self._thread_event_queue.get, True, timeout)
         except queue.Empty:
             if not self._thread.is_alive():
-                logger.error("Threaded server failed to start and is not alive. No error event was received.")
-                return
-            logger.error("Threaded server failed to start and sends no event. This should not happen.")
+                raise RuntimeError("Threaded server failed to start and is not alive. No error event was received.")
+            logger.error(
+                "Threaded server failed to start and sends no event. This should not happen. Shutting down server."
+            )
             await self._stop_uvicorn_thread()
-            return
+            raise RuntimeError("Threaded server failed to start and sends no event. This should not happen.")
 
         if evt.kind == "error":
             logger.error("Threaded server failed to start (%s): %s\n%s", evt.exc_type, evt.message, evt.traceback)
             await asyncio.to_thread(self._thread.join, self.args.thread_join_timeout)
             if self._thread.is_alive():
-                raise RuntimeError(evt.message or "Threaded server failed to start and refused to shut down.")
+                logger.error("Threaded server failed to start and refused to shut down.")
+            raise RuntimeError(evt.message)
         else:
             logger.info("Threaded server started successfully.")
             self._is_running = True
@@ -893,13 +932,18 @@ class PythonServerLauncher:
                 "workers": int(self.args.n_workers),
                 "worker_class": "uvicorn_worker.UvicornWorker",
                 "loglevel": logging.getLevelName(self.args.log_level).lower(),
-                "accesslog": None,
+                "accesslog": "-" if self.args.access_log else None,
                 "errorlog": "-",
                 "preload_app": True,
                 "graceful_timeout": int(
                     self.args.process_join_timeout / 2
                 ),  # Allow half the timeout for graceful shutdown
             }
+            if "PROMETHEUS_MULTIPROC_DIR" in os.environ:
+                from prometheus_client import multiprocess
+
+                options["child_exit"] = lambda server, worker: multiprocess.mark_process_dead(worker.pid)  # type: ignore
+
             self._gunicorn_app = GunicornApp(self.app, options)
 
             self._proc = ctx.Process(
@@ -939,11 +983,12 @@ class PythonServerLauncher:
             evt: ChildEvent = await asyncio.to_thread(self._mp_event_queue.get, True, timeout)
         except queue.Empty:
             if not self._proc.is_alive():
-                logger.error("Server process failed to start and is not alive. No error event was received.")
-                return
-            logger.error("Server process failed to start and sends no event. This should not happen.")
+                raise RuntimeError("Server process failed to start and is not alive. No error event was received.")
+            logger.error(
+                "Server process failed to start and sends no event. This should not happen. Shutting down server."
+            )
             await self._stop_serving_process()
-            return
+            raise RuntimeError("Server process failed to start and sends no event. This should not happen.")
 
         if evt.kind == "error":
             logger.error(
@@ -955,7 +1000,8 @@ class PythonServerLauncher:
             )
             await asyncio.to_thread(self._proc.join, self.args.process_join_timeout)
             if self._proc.is_alive():
-                raise RuntimeError(evt.message or "Server process failed to start and refused to shut down.")
+                logger.error("Server process failed to start and refused to shut down.")
+            raise RuntimeError(evt.message)
         else:
             logger.info("Subprocess server started successfully.")
             self._is_running = True

agentlightning/utils/system_snapshot.py

@@ -0,0 +1,72 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from __future__ import annotations
+
+import platform
+import socket
+from contextlib import suppress
+from datetime import datetime
+from typing import Any, Dict, List, cast
+
+import psutil
+from gpustat import GPUStat, GPUStatCollection
+
+
+def system_snapshot(include_gpu: bool = False) -> Dict[str, Any]:
+    # CPU
+    cpu = {
+        "cpu_name": platform.processor(),
+        "cpu_cores": psutil.cpu_count(logical=False),
+        "cpu_threads": psutil.cpu_count(logical=True),
+        "cpu_usage_pct": psutil.cpu_percent(0.05),
+    }
+
+    # Memory
+    vm = psutil.virtual_memory()
+    mem = {
+        "mem_used_gb": round(vm.used / (2**30), 2),
+        "mem_total_gb": round(vm.total / (2**30), 2),
+        "mem_pct": vm.percent,
+    }
+
+    # Disk
+    du = psutil.disk_usage("/")
+    disk = {
+        "disk_used_gb": round(du.used / (2**30), 2),
+        "disk_total_gb": round(du.total / (2**30), 2),
+        "disk_pct": du.percent,
+    }
+
+    # GPU
+    gpus: List[Dict[str, Any]] = []
+    with suppress(Exception):
+        for g in GPUStatCollection.new_query().gpus:  # type: ignore
+            g = cast(GPUStat, g)
+            gpus.append(
+                {
+                    "gpu": g.name,  # type: ignore
+                    "util_pct": g.utilization,
+                    "mem_used_mb": g.memory_used,
+                    "mem_total_mb": g.memory_total,
+                    "temp_c": g.temperature,
+                }
+            )
+
+    # Network
+    net = psutil.net_io_counters()
+    netinfo = {
+        "bytes_sent_mb": round(net.bytes_sent / (2**20), 2),
+        "bytes_recv_mb": round(net.bytes_recv / (2**20), 2),
+    }
+
+    # OS / meta
+    return {
+        "timestamp": datetime.now().isoformat(timespec="seconds"),
+        "host": socket.gethostname(),
+        "os": platform.platform(),
+        **cpu,
+        **mem,
+        **disk,
+        **netinfo,
+        **({"gpus": gpus} if include_gpu else {}),
+    }

agentlightning/verl/daemon.py

@@ -9,7 +9,7 @@ import time
 import uuid
 from collections import defaultdict
 from collections.abc import Mapping
-from typing import Any, Dict, List, Literal, Optional, Tuple
+from typing import Any, Dict, List, Literal, Optional, Tuple, cast
 
 import numpy as np
 import requests
@@ -18,13 +18,11 @@ from flask import Flask, Response, abort, request
 from tensordict import TensorDict
 from verl import DataProto
 
-from agentlightning import LLM, AgentLightningServer, NamedResources, RolloutLegacy, configure_logger
+from agentlightning import LLM, AgentLightningServer, NamedResources, RolloutLegacy
 from agentlightning.adapter.triplet import TracerTraceToTriplet, TraceToTripletBase
 from agentlightning.llm_proxy import LLMProxy, ModelConfig
 from agentlightning.store.base import LightningStore
-from agentlightning.types import Rollout, RolloutConfig, Task
-
-configure_logger()
+from agentlightning.types import EnqueueRolloutRequest, Rollout, RolloutConfig, Task
 
 __all__ = [
     "AgentModeDaemon",
@@ -379,42 +377,57 @@ class AgentModeDaemon:
         num_samples = len(data[keys[0]])
         rollouts_per_sample = self.train_rollout_n if is_train else 1
 
+        enqueue_rollout_requests: List[EnqueueRolloutRequest] = []
+        data_id_to_original_sample: Dict[str, Dict[str, Any]] = {}
+
         for i in range(num_samples):
             data_id = str(uuid.uuid4())
             original_sample = {key: data[key][i] for key in keys}
             original_sample["data_id"] = data_id
+            data_id_to_original_sample[data_id] = original_sample
 
             # For training, each sample is rolled out multiple times
+            # Data ID is different from Rollout ID, as one data can have multiple rollouts.
             for _ in range(rollouts_per_sample):
                 task_metadata = {"data_id": data_id, "is_train": is_train}
-
-                # Data ID is different from Rollout ID, as one data can have multiple rollouts.
                 if self.mode == "v0":
+                    # Queue immediately
                     rollout_id = await self.server.queue_task(
                         sample=_to_native(original_sample),
                         mode="train" if is_train else "val",
                         resources_id=resources_id,
                         metadata=task_metadata,
                     )
-                else:
-                    rollout = await self.store.enqueue_rollout(
-                        input=_to_native(original_sample),
-                        mode="train" if is_train else "val",
-                        resources_id=resources_id,
-                        metadata=task_metadata,
-                    )
-                    await self.store.update_rollout(
-                        rollout_id=rollout.rollout_id,
-                        config=RolloutConfig(
-                            unresponsive_seconds=self.llm_timeout_seconds,
-                            timeout_seconds=self.llm_timeout_seconds,
-                        ),
-                    )
-                    rollout_id = rollout.rollout_id
 
-                # Store original sample data to reconstruct batch information later
-                self._task_id_to_original_sample[rollout_id] = original_sample
-                self._total_tasks_queued += 1
+                    # Store original sample data to reconstruct batch information later
+                    self._task_id_to_original_sample[rollout_id] = original_sample
+                    self._total_tasks_queued += 1
+                else:
+                    # Collect tasks to enqueue in batch and queue them later
+                    enqueue_rollout_requests.append(
+                        EnqueueRolloutRequest(
+                            input=_to_native(original_sample),
+                            mode="train" if is_train else "val",
+                            resources_id=resources_id,
+                            config=RolloutConfig(
+                                unresponsive_seconds=self.llm_timeout_seconds,
+                                timeout_seconds=self.llm_timeout_seconds,
+                            ),
+                            metadata=task_metadata,
+                        )
+                    )
+
+        if self.mode == "v1":
+            # Enqueue all the tasks in a single batch
+            rollouts = await self.store.enqueue_many_rollouts(enqueue_rollout_requests)
+            self._task_id_to_original_sample.update(
+                {
+                    # Recover the original data and store it for later use.
+                    rollout.rollout_id: data_id_to_original_sample[cast(Dict[str, Any], rollout.metadata)["data_id"]]
+                    for rollout in rollouts
+                }
+            )
+            self._total_tasks_queued += len(rollouts)
 
     def set_up_data_and_server(self, data: Dict[str, Any], server_addresses: List[str], is_train: bool = True):
         """Synchronous wrapper for setting up data and server resources."""
@@ -559,14 +572,17 @@ class AgentModeDaemon:
         )  # FIXME: Evaluate whether grouping stats by source is actually needed.
 
         for rollout_id, rollout in self._completed_rollouts_v0.items():
+            final_reward_raw: Optional[float] = rollout.final_reward
             final_reward = self._fillna_reward(rollout)
             if not rollout.triplets:
                 print(f"Warning: No triplets found for test rollout {rollout.rollout_id}.")
-                sample_stat_list.append({"reward": final_reward})
+                sample_stat_list.append({"reward": final_reward, "has_reward": final_reward_raw is not None})
                 continue
             response_length_list = [len(triplet.response.get("token_ids", [])) for triplet in rollout.triplets]
+
             if "data_source" in self._task_id_to_original_sample[rollout_id]:
                 # When a test sample includes a 'data_source' field, record per-source statistics for test results.
+                # TODO: This is a flawed design. We should have a better way to handle this.
                 data_source = self._task_id_to_original_sample[rollout_id]["data_source"]
                 sample_stat_list_by_source[data_source].append(
                     {
@@ -574,6 +590,7 @@ class AgentModeDaemon:
                         "mean_response_length": np.mean(response_length_list) if response_length_list else 0,
                         "turn_count": len(rollout.triplets),
                         "reward": final_reward,
+                        "has_reward": final_reward_raw is not None,
                     }
                 )
             sample_stat_list.append(
@@ -582,6 +599,7 @@ class AgentModeDaemon:
                     "mean_response_length": np.mean(response_length_list) if response_length_list else 0,
                     "turn_count": len(rollout.triplets),
                     "reward": final_reward,
+                    "has_reward": final_reward_raw is not None,
                 }
             )
         metric_dict: Dict[str, Any] = {}
@@ -596,6 +614,9 @@ class AgentModeDaemon:
                 {
                     f"val/{data_source}/n_rollouts": len(sample_stats),
                     f"val/{data_source}/n_rollouts_w_trace": len(stats_w_trace_by_source[data_source]),
+                    f"val/{data_source}/n_rollouts_w_reward": len(
+                        [stat for stat in sample_stats if stat["has_reward"]]
+                    ),
                     f"val/{data_source}/reward": np.mean(
                         [stat["reward"] for stat in sample_stats]
                     ),  # each rollout must have a reward (fillna if missing)
@@ -614,6 +635,7 @@ class AgentModeDaemon:
             {
                 "val/n_rollouts": len(sample_stat_list),
                 "val/n_rollouts_w_trace": len(stats_w_trace),
+                "val/n_rollouts_w_reward": len([stat for stat in sample_stat_list if stat["has_reward"]]),
                 "val/reward": np.mean(
                     [stat["reward"] for stat in sample_stat_list]
                 ),  # each rollout must have a reward (fillna if missing)
@@ -638,9 +660,10 @@ class AgentModeDaemon:
         # 1. Reconstruct the `finished_id_to_sample_info` structure from completed rollouts
         finished_id_to_sample_info: Dict[str, Dict[str, Any]] = {}
         finished_id_to_final_reward: Dict[str, float] = {}
+        sample_with_reward_count = 0
         for rollout_id, rollout in self._completed_rollouts_v0.items():
             original_sample = self._task_id_to_original_sample[rollout_id]
-
+            sample_with_reward_count += int(rollout.final_reward is not None)
             final_reward = self._fillna_reward(rollout)
 
             if not rollout.triplets:
@@ -759,6 +782,7 @@ class AgentModeDaemon:
             "training/reward": np.mean(list(finished_id_to_final_reward.values())),
             "training/n_rollouts": len(finished_id_to_final_reward),
             "training/n_rollouts_w_trace": len(finished_id_to_sample_info),
+            "training/n_rollouts_w_reward": sample_with_reward_count,
             "training/n_truncated_triplets": n_trunc_sample_because_of_response,
             "training/n_triplets": n_transition,
         }

agentlightning/verl/trainer.py

@@ -12,6 +12,7 @@ from typing import Dict, Tuple
 
 import numpy as np
 import torch
+import verl
 from codetiming import Timer
 from omegaconf import OmegaConf
 from tqdm import tqdm
@@ -403,14 +404,20 @@ class AgentLightningTrainer(RayPPOTrainer):
         assert self.async_rollout_mode, "If agent mode is enabled, async server must be enabled"
         if self.adapter is not None and not isinstance(self.adapter, TraceToTripletBase):
             raise ValueError("Adapter must be a TraceToTripletBase for currently VERL implementation.")
+        verl_version = verl.__version__
+        if verl_version == "0.5.0":
+            # Note (Zhiyuan): To avoid further patch into vllm async server, using the same sentence to get the naming here.
+            # However, it is possible that verl updates the naming and causes incompatibility.
+            # Reference: https://github.com/volcengine/verl/blob/5b5e09d9cc20625e436d01f69d9cc739ff681c54/verl/workers/rollout/vllm_rollout/vllm_async_server.py#L217
+            model = "/".join(self.config.actor_rollout_ref.model.path.split("/")[-2:])
+        else:
+            # For other versions (e.g., 0.6.0), we use the full path to the model.
+            model = self.config.actor_rollout_ref.model.path
         self.agent_mode_daemon = AgentModeDaemon(
             self.config.agentlightning.port,
             self.config.actor_rollout_ref.rollout.n,
             train_information={
-                # Note (Zhiyuan): To avoid further patch into vllm async server, using the same sentence to get the naming here.
-                # However, it is possible that verl updates the naming and causes incompatibility.
-                # Reference: https://github.com/volcengine/verl/blob/5b5e09d9cc20625e436d01f69d9cc739ff681c54/verl/workers/rollout/vllm_rollout/vllm_async_server.py#L217
-                "model": "/".join(self.config.actor_rollout_ref.model.path.split("/")[-2:]),
+                "model": model,
                 "temperature": self.config.actor_rollout_ref.rollout.temperature,
             },
             tokenizer=self.tokenizer,

dashboard/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "agent-lightning-dashboard",
-  "version": "0.2.2",
+  "version": "0.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "agent-lightning-dashboard",
-      "version": "0.2.2",
+      "version": "0.3.0",
       "dependencies": {
         "@mantine/core": "8.3.5",
         "@mantine/hooks": "8.3.5",

dashboard/package.json

@@ -1,7 +1,7 @@
 {
   "name": "agent-lightning-dashboard",
   "type": "module",
-  "version": "0.2.2",
+  "version": "0.3.0",
   "scripts": {
     "dev": "vite",
     "build": "tsc && vite build",

dashboard/src/Router.tsx

@@ -6,6 +6,7 @@ import { ResourcesPage } from './pages/Resources.page';
 import { RolloutsPage } from './pages/Rollouts.page';
 import { SettingsPage } from './pages/Settings.page';
 import { TracesPage } from './pages/Traces.page';
+import { WorkersPage } from './pages/Workers.page';
 
 const router = createBrowserRouter([
   {
@@ -28,6 +29,10 @@ const router = createBrowserRouter([
         path: 'traces',
         element: <TracesPage />,
       },
+      {
+        path: 'runners',
+        element: <WorkersPage />,
+      },
       {
         path: 'settings',
         element: <SettingsPage />,

dashboard/src/components/AppDrawer.component.tsx

@@ -21,7 +21,7 @@ import {
 import { useGetSpansQuery } from '@/features/rollouts';
 import { closeDrawer, openDrawer, selectDrawerContent, selectDrawerIsOpen } from '@/features/ui/drawer';
 import { useAppDispatch, useAppSelector } from '@/store/hooks';
-import type { Attempt, AttemptStatus, Rollout, RolloutStatus, Span } from '@/types';
+import type { Attempt, AttemptStatus, Rollout, RolloutStatus, Span, Worker } from '@/types';
 import { formatStatusLabel } from '@/utils/format';
 import { TracesTable, type TracesTableRecord } from './TracesTable.component';
 
@@ -50,6 +50,12 @@ const SPAN_STATUS_COLORS: Record<Span['status']['status_code'], string> = {
   ERROR: 'red',
 };
 
+const WORKER_STATUS_COLORS: Record<Worker['status'], string> = {
+  busy: 'orange',
+  idle: 'teal',
+  unknown: 'gray',
+};
+
 const TRACES_SORT_FIELD_MAP: Record<string, string> = {
   name: 'name',
   traceId: 'trace_id',
@@ -408,6 +414,60 @@ function RolloutTracesDrawerBody({ rollout, attempt, onShowRollout, onShowSpanDe
   );
 }
 
+type WorkerDrawerTitleProps = {
+  worker: Worker;
+};
+
+function WorkerDrawerTitle({ worker }: WorkerDrawerTitleProps) {
+  const badgeColor = WORKER_STATUS_COLORS[worker.status] ?? 'gray';
+  return (
+    <Stack gap={3}>
+      <Group gap={6} align='center'>
+        <Text fw={600}>{worker.workerId}</Text>
+        <CopyButton value={worker.workerId}>
+          {({ copied, copy }) => (
+            <Tooltip label={copied ? 'Copied' : 'Copy'} withArrow>
+              <ActionIcon
+                aria-label={`Copy worker ID ${worker.workerId}`}
+                variant='subtle'
+                color={copied ? 'teal' : 'gray'}
+                size='sm'
+                onClick={(event) => {
+                  event.stopPropagation();
+                  copy();
+                }}
+              >
+                {copied ? <IconCheck size={14} /> : <IconCopy size={14} />}
+              </ActionIcon>
+            </Tooltip>
+          )}
+        </CopyButton>
+        <Badge size='sm' variant='light' color={badgeColor}>
+          {formatStatusLabel(worker.status)}
+        </Badge>
+      </Group>
+      <Group gap='xl'>
+        <Group gap={4}>
+          <Text size='sm' c='dimmed' fw={500}>
+            Rollout
+          </Text>
+          <Text size='sm' c='dimmed'>
+            {worker.currentRolloutId ?? '—'}
+          </Text>
+        </Group>
+        <Group gap={4}>
+          <Text size='sm' c='dimmed' fw={500}>
+            Attempt
+          </Text>
+          <Text size='sm' c='dimmed'>
+            {worker.currentAttemptId ?? '—'}
+          </Text>
+        </Group>
+      </Group>
+    </Stack>
+  );
+}
+
 export function AppDrawerContainer() {
   const dispatch = useAppDispatch();
   const isOpen = useAppSelector(selectDrawerIsOpen);
@@ -428,6 +488,13 @@ export function AppDrawerContainer() {
       return null;
     }
 
+    if (content.type === 'worker-detail') {
+      const { worker } = content;
+      const title = <WorkerDrawerTitle worker={worker} />;
+      const body = <JsonEditor value={worker} />;
+      return { title, body };
+    }
+
     if (content.type === 'trace-detail') {
       const { span } = content;
       const title = <TraceDrawerTitle span={span} />;

dashboard/src/components/TracesTable.component.tsx

@@ -22,6 +22,7 @@ const DEFAULT_RECORDS_PER_PAGE_OPTIONS = [50, 100, 200, 500];
 
 const COLUMN_VISIBILITY: Record<string, ColumnVisibilityConfig> = {
   name: { minWidth: 12.5, priority: 0 },
+  sequenceId: { fixedWidth: 6, priority: 1 },
   spanId: { fixedWidth: 14, priority: 1 },
   traceId: { fixedWidth: 24, priority: 3 },
   parentId: { fixedWidth: 12, priority: 2 },
@@ -86,6 +87,12 @@ function createTracesColumns({
         </Text>
       ),
     },
+    {
+      accessor: 'sequenceId',
+      title: 'Seq.',
+      sortable: true,
+      render: ({ sequenceId }) => <Text size='sm'>{sequenceId}</Text>,
+    },
     {
       accessor: 'traceId',
       title: 'Trace ID',

dashboard/src/components/WorkersTable.component.tsx

@@ -0,0 +1,362 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+import { useCallback, useEffect, useMemo } from 'react';
+import { IconCheck, IconCopy, IconInfoCircle, IconRefresh } from '@tabler/icons-react';
+import { DataTable, type DataTableColumn, type DataTableSortStatus } from 'mantine-datatable';
+import { ActionIcon, Badge, Box, Button, CopyButton, Group, Stack, Text, Tooltip } from '@mantine/core';
+import { useElementSize, useViewportSize } from '@mantine/hooks';
+import { getLayoutAwareWidth } from '@/layouts/helper';
+import type { Worker } from '@/types';
+import { getErrorDescriptor } from '@/utils/error';
+import { formatDateTime, formatRelativeTime, formatStatusLabel } from '@/utils/format';
+import { createResponsiveColumns, type ColumnVisibilityConfig } from '@/utils/table';
+
+const DEFAULT_RECORDS_PER_PAGE_OPTIONS = [50, 100, 200, 500];
+
+const COLUMN_VISIBILITY: Record<string, ColumnVisibilityConfig> = {
+  workerId: { fixedWidth: 12, priority: 0 },
+  status: { fixedWidth: 6, priority: 1 },
+  currentRolloutId: { fixedWidth: 14, priority: 3 },
+  currentAttemptId: { fixedWidth: 14, priority: 3 },
+  lastHeartbeatTime: { fixedWidth: 10, priority: 2 },
+  lastBusyTime: { fixedWidth: 10, priority: 3 },
+  lastIdleTime: { fixedWidth: 10, priority: 3 },
+  lastDequeueTime: { fixedWidth: 10, priority: 1 },
+  actions: { fixedWidth: 5, priority: 0 },
+};
+
+export type WorkersTableRecord = Worker & {
+  timestamps: Record<
+    'lastHeartbeatTime' | 'lastBusyTime' | 'lastIdleTime' | 'lastDequeueTime',
+    { absolute: string; relative: string }
+  >;
+};
+
+const buildTimestampMeta = (value: Worker['lastHeartbeatTime']) => ({
+  absolute: formatDateTime(value),
+  relative: formatRelativeTime(value),
+});
+
+function buildWorkerRecord(worker: Worker): WorkersTableRecord {
+  return {
+    ...worker,
+    timestamps: {
+      lastHeartbeatTime: buildTimestampMeta(worker.lastHeartbeatTime),
+      lastBusyTime: buildTimestampMeta(worker.lastBusyTime),
+      lastIdleTime: buildTimestampMeta(worker.lastIdleTime),
+      lastDequeueTime: buildTimestampMeta(worker.lastDequeueTime),
+    },
+  };
+}
+
+type WorkersColumnsOptions = {
+  onShowDetails: (worker: Worker) => void;
+};
+
+const STATUS_COLORS: Record<Worker['status'], string> = {
+  busy: 'orange',
+  idle: 'teal',
+  unknown: 'gray',
+};
+
+function createWorkersColumns({ onShowDetails }: WorkersColumnsOptions): DataTableColumn<WorkersTableRecord>[] {
+  return [
+    {
+      accessor: 'workerId',
+      title: 'Runner ID',
+      sortable: true,
+      render: ({ workerId }) => (
+        <Group gap={2} wrap='nowrap'>
+          <Text fw={500} size='sm'>
+            {workerId}
+          </Text>
+          <CopyButton value={workerId}>
+            {({ copied, copy }) => (
+              <Tooltip label={copied ? 'Copied' : 'Copy'} withArrow>
+                <ActionIcon
+                  aria-label={`Copy worker ID ${workerId}`}
+                  variant='subtle'
+                  color={copied ? 'teal' : 'gray'}
+                  size='sm'
+                  onClick={(event) => {
+                    event.stopPropagation();
+                    copy();
+                  }}
+                >
+                  {copied ? <IconCheck size={14} /> : <IconCopy size={14} />}
+                </ActionIcon>
+              </Tooltip>
+            )}
+          </CopyButton>
+        </Group>
+      ),
+    },
+    {
+      accessor: 'status',
+      title: 'Status',
+      sortable: true,
+      render: ({ status }) => {
+        const color = STATUS_COLORS[status] ?? 'gray';
+        return (
+          <Badge size='sm' variant='light' color={color} radius='sm'>
+            {formatStatusLabel(status)}
+          </Badge>
+        );
+      },
+    },
+    {
+      accessor: 'currentRolloutId',
+      title: 'Current Rollout',
+      sortable: true,
+      render: ({ currentRolloutId }) => <Text size='sm'>{currentRolloutId ?? '—'}</Text>,
+    },
+    {
+      accessor: 'currentAttemptId',
+      title: 'Current Attempt',
+      sortable: true,
+      render: ({ currentAttemptId }) => <Text size='sm'>{currentAttemptId ?? '—'}</Text>,
+    },
+    {
+      accessor: 'lastHeartbeatTime',
+      title: 'Heartbeat',
+      sortable: true,
+      render: ({ timestamps }) => (
+        <Stack gap={0} justify='center'>
+          <Text size='sm'>{timestamps.lastHeartbeatTime.relative}</Text>
+          {timestamps.lastHeartbeatTime.absolute !== '—' && (
+            <Text size='xs' c='dimmed'>
+              {timestamps.lastHeartbeatTime.absolute}
+            </Text>
+          )}
+        </Stack>
+      ),
+    },
+    {
+      accessor: 'lastBusyTime',
+      title: 'Last Busy',
+      sortable: true,
+      render: ({ timestamps }) => (
+        <Stack gap={0} justify='center'>
+          <Text size='sm'>{timestamps.lastBusyTime.relative}</Text>
+          {timestamps.lastBusyTime.absolute !== '—' && (
+            <Text size='xs' c='dimmed'>
+              {timestamps.lastBusyTime.absolute}
+            </Text>
+          )}
+        </Stack>
+      ),
+    },
+    {
+      accessor: 'lastIdleTime',
+      title: 'Last Idle',
+      sortable: true,
+      render: ({ timestamps }) => (
+        <Stack gap={0} justify='center'>
+          <Text size='sm'>{timestamps.lastIdleTime.relative}</Text>
+          {timestamps.lastIdleTime.absolute !== '—' && (
+            <Text size='xs' c='dimmed'>
+              {timestamps.lastIdleTime.absolute}
+            </Text>
+          )}
+        </Stack>
+      ),
+    },
+    {
+      accessor: 'lastDequeueTime',
+      title: 'Last Dequeue',
+      sortable: true,
+      render: ({ timestamps }) => (
+        <Stack gap={0} justify='center'>
+          <Text size='sm'>{timestamps.lastDequeueTime.relative}</Text>
+          {timestamps.lastDequeueTime.absolute !== '—' && (
+            <Text size='xs' c='dimmed'>
+              {timestamps.lastDequeueTime.absolute}
+            </Text>
+          )}
+        </Stack>
+      ),
+    },
+    {
+      accessor: 'actions',
+      title: 'Actions',
+      textAlign: 'left',
+      render: (record) => (
+        <Tooltip label='Show runner detail' withArrow disabled={!onShowDetails}>
+          <ActionIcon
+            aria-label='Show runner detail'
+            variant='subtle'
+            color='gray'
+            onClick={(event) => {
+              event.stopPropagation();
+              onShowDetails(record);
+            }}
+          >
+            <IconInfoCircle size={16} />
+          </ActionIcon>
+        </Tooltip>
+      ),
+    },
+  ];
+}
+
+export type WorkersTableProps = {
+  workers: Worker[] | undefined;
+  totalRecords: number;
+  isFetching: boolean;
+  isError: boolean;
+  error: unknown;
+  searchTerm: string;
+  sort: { column: string; direction: 'asc' | 'desc' };
+  page: number;
+  recordsPerPage: number;
+  onSortStatusChange: (status: DataTableSortStatus<WorkersTableRecord>) => void;
+  onPageChange: (page: number) => void;
+  onRecordsPerPageChange: (value: number) => void;
+  onResetFilters: () => void;
+  onRefetch: () => void;
+  onShowDetails: (worker: Worker) => void;
+  recordsPerPageOptions?: number[];
+};
+
+export function WorkersTable({
+  workers,
+  totalRecords,
+  isFetching,
+  isError,
+  error,
+  searchTerm,
+  sort,
+  page,
+  recordsPerPage,
+  onSortStatusChange,
+  onPageChange,
+  onRecordsPerPageChange,
+  onResetFilters,
+  onRefetch,
+  onShowDetails,
+  recordsPerPageOptions = DEFAULT_RECORDS_PER_PAGE_OPTIONS,
+}: WorkersTableProps) {
+  const { ref: tableContainerRef, width: containerWidth } = useElementSize();
+  const { width: viewportWidth } = useViewportSize();
+
+  const layoutAwareContainerWidth = useMemo(
+    () => getLayoutAwareWidth(containerWidth, viewportWidth),
+    [containerWidth, viewportWidth],
+  );
+
+  const workerRecords = useMemo<WorkersTableRecord[]>(() => {
+    if (!workers) {
+      return [];
+    }
+    return workers.map((worker) => buildWorkerRecord(worker));
+  }, [workers]);
+
+  const columns = useMemo(() => createWorkersColumns({ onShowDetails }), [onShowDetails]);
+  const responsiveColumns = useMemo(
+    () => createResponsiveColumns(columns, layoutAwareContainerWidth, COLUMN_VISIBILITY),
+    [columns, layoutAwareContainerWidth],
+  );
+
+  const totalPages = useMemo(
+    () => Math.max(1, Math.ceil(Math.max(0, totalRecords) / Math.max(1, recordsPerPage))),
+    [recordsPerPage, totalRecords],
+  );
+
+  useEffect(() => {
+    if (page > totalPages) {
+      onPageChange(totalPages);
+    }
+  }, [onPageChange, page, totalPages]);
+
+  const hasActiveFilters = searchTerm.trim().length > 0;
+
+  const sortStatus: DataTableSortStatus<WorkersTableRecord> = {
+    columnAccessor: sort.column,
+    direction: sort.direction,
+  };
+
+  const handleSortStatusChange = useCallback(
+    (status: DataTableSortStatus<WorkersTableRecord>) => {
+      onSortStatusChange(status);
+    },
+    [onSortStatusChange],
+  );
+
+  const errorDescriptor = isError ? getErrorDescriptor(error) : null;
+  const errorMessage = isError
+    ? `Workers are temporarily unavailable${errorDescriptor ? ` (${errorDescriptor})` : ''}.`
+    : 'Workers are temporarily unavailable.';
+
+  const emptyState = (
+    <Stack gap='sm' align='center' py='lg'>
+      {isError ? (
+        <>
+          <Text fw={600} size='sm'>
+            {errorMessage}
+          </Text>
+          <Text size='sm' c='dimmed' ta='center'>
+            Use the retry button to try again, or adjust the search to broaden the results.
+          </Text>
+          <Group gap='xs'>
+            <Button size='xs' variant='light' color='gray' leftSection={<IconRefresh size={14} />} onClick={onRefetch}>
+              Retry
+            </Button>
+            {hasActiveFilters ? (
+              <Button size='xs' variant='subtle' onClick={onResetFilters}>
+                Clear filters
+              </Button>
+            ) : null}
+          </Group>
+        </>
+      ) : (
+        <>
+          <Text fw={600} size='sm'>
+            No workers found
+          </Text>
+          <Text size='sm' c='dimmed' ta='center'>
+            {hasActiveFilters
+              ? 'Try adjusting the search to see more results.'
+              : 'Try refreshing to fetch the latest worker status.'}
+          </Text>
+          <Group gap='xs'>
+            <Button size='xs' variant='light' leftSection={<IconRefresh size={14} />} onClick={onRefetch}>
+              Refresh
+            </Button>
+            {hasActiveFilters ? (
+              <Button size='xs' variant='subtle' onClick={onResetFilters}>
+                Clear filters
+              </Button>
+            ) : null}
+          </Group>
+        </>
+      )}
+    </Stack>
+  );
+
+  return (
+    <Box ref={tableContainerRef}>
+      <DataTable<WorkersTableRecord>
+        classNames={{ root: 'workers-table' }}
+        withTableBorder
+        withColumnBorders
+        highlightOnHover
+        verticalAlign='center'
+        minHeight={workerRecords.length === 0 ? 400 : undefined}
+        idAccessor='workerId'
+        records={workerRecords}
+        columns={responsiveColumns}
+        totalRecords={totalRecords}
+        recordsPerPage={recordsPerPage}
+        page={page}
+        onPageChange={onPageChange}
+        onRecordsPerPageChange={onRecordsPerPageChange}
+        recordsPerPageOptions={recordsPerPageOptions}
+        sortStatus={sortStatus}
+        onSortStatusChange={handleSortStatusChange}
+        fetching={isFetching}
+        loaderSize='sm'
+        emptyState={workerRecords.length === 0 ? emptyState : undefined}
+      />
+    </Box>
+  );
+}

dashboard/src/components/WorkersTable.story.tsx

@@ -0,0 +1,154 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+import { useMemo, useState } from 'react';
+import type { Meta, StoryObj } from '@storybook/react';
+import { IconSearch } from '@tabler/icons-react';
+import { Box, Stack, TextInput, Title } from '@mantine/core';
+import type { Worker } from '@/types';
+import { WorkersTable } from './WorkersTable.component';
+
+const meta: Meta<typeof WorkersTable> = {
+  title: 'Components/WorkersTable',
+  component: WorkersTable,
+  parameters: {
+    layout: 'fullscreen',
+  },
+};
+
+export default meta;
+
+type Story = StoryObj<typeof WorkersTable>;
+
+const now = Math.floor(Date.now() / 1000);
+
+const sampleWorkers: Worker[] = [
+  {
+    workerId: 'worker-east',
+    status: 'busy',
+    heartbeatStats: { queueDepth: 2, gpuUtilization: 0.82 },
+    lastHeartbeatTime: now - 20,
+    lastDequeueTime: now - 60,
+    lastBusyTime: now - 120,
+    lastIdleTime: now - 600,
+    currentRolloutId: 'ro-story-001',
+    currentAttemptId: 'at-story-010',
+  },
+  {
+    workerId: 'worker-west',
+    status: 'busy',
+    heartbeatStats: { queueDepth: 1 },
+    lastHeartbeatTime: now - 45,
+    lastDequeueTime: now - 300,
+    lastBusyTime: now - 200,
+    lastIdleTime: now - 4800,
+    currentRolloutId: 'ro-story-003',
+    currentAttemptId: 'at-story-033',
+  },
+  {
+    workerId: 'worker-north',
+    status: 'idle',
+    heartbeatStats: { queueDepth: 0 },
+    lastHeartbeatTime: now - 90,
+    lastDequeueTime: now - 3600,
+    lastBusyTime: now - 5400,
+    lastIdleTime: now - 5400,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+  {
+    workerId: 'worker-south',
+    status: 'idle',
+    heartbeatStats: null,
+    lastHeartbeatTime: now - 900,
+    lastDequeueTime: now - 7200,
+    lastBusyTime: now - 8600,
+    lastIdleTime: now - 8600,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+  {
+    workerId: 'worker-standby',
+    status: 'unknown',
+    heartbeatStats: { queueDepth: 0 },
+    lastHeartbeatTime: now - 15,
+    lastDequeueTime: now - 4000,
+    lastBusyTime: null,
+    lastIdleTime: null,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+];
+
+type WorkersTableStoryWrapperProps = {
+  maxWidth: number;
+  initialSort?: { column: string; direction: 'asc' | 'desc' };
+};
+
+function WorkersTableStoryWrapper({ maxWidth, initialSort }: WorkersTableStoryWrapperProps) {
+  const [searchTerm, setSearchTerm] = useState('');
+  const [page, setPage] = useState(1);
+  const [recordsPerPage, setRecordsPerPage] = useState(5);
+  const [sort, setSort] = useState<{ column: string; direction: 'asc' | 'desc' }>(
+    () => initialSort ?? { column: 'lastHeartbeatTime', direction: 'desc' },
+  );
+
+  const filteredWorkers = useMemo(() => {
+    const normalized = searchTerm.trim().toLowerCase();
+    if (!normalized) {
+      return sampleWorkers;
+    }
+    return sampleWorkers.filter((worker) => worker.workerId.toLowerCase().includes(normalized));
+  }, [searchTerm]);
+
+  return (
+    <Stack gap='md' p='lg'>
+      <Title order={2}>Workers ({maxWidth}px max width)</Title>
+      <TextInput
+        placeholder='Search'
+        leftSection={<IconSearch size={16} />}
+        value={searchTerm}
+        onChange={(event) => setSearchTerm(event.currentTarget.value)}
+        w='100%'
+        style={{ maxWidth: 360 }}
+      />
+      <Box style={{ maxWidth }}>
+        <WorkersTable
+          workers={filteredWorkers}
+          totalRecords={filteredWorkers.length}
+          isFetching={false}
+          isError={false}
+          error={null}
+          searchTerm={searchTerm}
+          sort={sort}
+          page={page}
+          recordsPerPage={recordsPerPage}
+          onSortStatusChange={(status) => {
+            return setSort({ column: status.columnAccessor as string, direction: status.direction });
+          }}
+          onPageChange={setPage}
+          onRecordsPerPageChange={setRecordsPerPage}
+          onResetFilters={() => {
+            setSearchTerm('');
+            setPage(1);
+          }}
+          onRefetch={() => {}}
+          onShowDetails={() => {}}
+        />
+      </Box>
+    </Stack>
+  );
+}
+
+export const Wide: Story = {
+  render: () => <WorkersTableStoryWrapper maxWidth={1600} />,
+};
+
+export const Narrow: Story = {
+  render: () => <WorkersTableStoryWrapper maxWidth={780} />,
+};
+
+export const SortedByCurrentRollout: Story = {
+  render: () => (
+    <WorkersTableStoryWrapper maxWidth={1200} initialSort={{ column: 'currentRolloutId', direction: 'asc' }} />
+  ),
+};

dashboard/src/features/rollouts/api.ts

@@ -13,6 +13,8 @@ import type {
   RolloutStatus,
   Span,
   Timestamp,
+  Worker,
+  WorkerStatus,
 } from '../../types';
 
 const rawBaseQuery = fetchBaseQuery({
@@ -122,6 +124,21 @@ const normalizeResources = (value: unknown): Resources => {
   };
 };
 
+const normalizeWorker = (value: unknown): Worker => {
+  const camelized = camelCaseKeys(value) as Worker;
+  return {
+    workerId: camelized.workerId,
+    status: camelized.status,
+    heartbeatStats: camelized.heartbeatStats ?? null,
+    lastHeartbeatTime: camelized.lastHeartbeatTime ?? null,
+    lastDequeueTime: camelized.lastDequeueTime ?? null,
+    lastBusyTime: camelized.lastBusyTime ?? null,
+    lastIdleTime: camelized.lastIdleTime ?? null,
+    currentRolloutId: camelized.currentRolloutId ?? null,
+    currentAttemptId: camelized.currentAttemptId ?? null,
+  };
+};
+
 const normalizePaginatedResponse = <T>(value: unknown, normalizer: (item: unknown) => T): PaginatedResponse<T> => {
   if (!value || typeof value !== 'object') {
     throw new Error('Expected paginated response payload');
@@ -183,6 +200,15 @@ export type GetResourcesQueryArgs = {
   resourcesIdContains?: string | null;
 };
 
+export type GetWorkersQueryArgs = {
+  limit: number;
+  offset: number;
+  sortBy?: string | null;
+  sortOrder?: 'asc' | 'desc';
+  workerIdContains?: string | null;
+  statusIn?: WorkerStatus[];
+};
+
 export type GetRolloutAttemptsQueryArgs = {
   rolloutId: string;
   limit?: number;
@@ -208,7 +234,7 @@ export type GetSpansQueryArgs = {
 export const rolloutsApi = createApi({
   reducerPath: 'rolloutsApi',
   baseQuery: dynamicBaseQuery,
-  tagTypes: ['Rollout', 'Span', 'Resources'],
+  tagTypes: ['Rollout', 'Span', 'Resources', 'Worker'],
   endpoints: (builder) => ({
     getResources: builder.query<PaginatedResponse<Resources>, GetResourcesQueryArgs>({
       query: ({ limit, offset, sortBy, sortOrder, resourcesIdContains }) => {
@@ -238,6 +264,37 @@ export const rolloutsApi = createApi({
             ]
           : [{ type: 'Resources' as const, id: 'LIST' }],
     }),
+    getWorkers: builder.query<PaginatedResponse<Worker>, GetWorkersQueryArgs>({
+      query: ({ limit, offset, sortBy, sortOrder, workerIdContains, statusIn }) => {
+        const searchParams = new URLSearchParams();
+        searchParams.set('limit', String(typeof limit === 'number' ? limit : -1));
+        searchParams.set('offset', String(typeof offset === 'number' ? offset : 0));
+        if (sortBy) {
+          searchParams.set('sort_by', sortBy);
+        }
+        if (sortOrder) {
+          searchParams.set('sort_order', sortOrder);
+        }
+        if (workerIdContains && workerIdContains.trim().length > 0) {
+          searchParams.set('worker_id_contains', workerIdContains.trim());
+        }
+        if (statusIn && statusIn.length > 0) {
+          statusIn.forEach((status) => searchParams.append('status_in', status));
+        }
+
+        const queryString = searchParams.toString();
+        const url = queryString.length > 0 ? `v1/agl/workers?${queryString}` : 'v1/agl/workers';
+        return { url, method: 'GET' };
+      },
+      transformResponse: (response: unknown) => normalizePaginatedResponse(response, normalizeWorker),
+      providesTags: (result) =>
+        result
+          ? [
+              { type: 'Worker' as const, id: 'LIST' },
+              ...result.items.map((worker) => ({ type: 'Worker' as const, id: worker.workerId })),
+            ]
+          : [{ type: 'Worker' as const, id: 'LIST' }],
+    }),
     getRollouts: builder.query<PaginatedResponse<Rollout>, GetRolloutsQueryArgs>({
       query: ({ limit, offset, sortBy, sortOrder, statusIn, rolloutIdContains, modeIn }) => {
         const searchParams = new URLSearchParams();
@@ -343,4 +400,10 @@ export const rolloutsApi = createApi({
   }),
 });
 
-export const { useGetResourcesQuery, useGetRolloutsQuery, useGetRolloutAttemptsQuery, useGetSpansQuery } = rolloutsApi;
+export const {
+  useGetResourcesQuery,
+  useGetWorkersQuery,
+  useGetRolloutsQuery,
+  useGetRolloutAttemptsQuery,
+  useGetSpansQuery,
+} = rolloutsApi;

dashboard/src/features/ui/drawer/slice.ts

@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 import { createSlice, type PayloadAction } from '@reduxjs/toolkit';
-import type { Attempt, Rollout, Span } from '@/types';
+import type { Attempt, Rollout, Span, Worker } from '@/types';
 
-export type DrawerType = 'rollout-json' | 'rollout-traces' | 'trace-detail';
+export type DrawerType = 'rollout-json' | 'rollout-traces' | 'trace-detail' | 'worker-detail';
 
 export type DrawerContent =
   | {
@@ -17,6 +17,10 @@ export type DrawerContent =
       span: Span;
       rollout: Rollout | null;
       attempt: Attempt | null;
+    }
+  | {
+      type: 'worker-detail';
+      worker: Worker;
     };
 
 export type DrawerState = {

dashboard/src/features/workers/index.ts

@@ -0,0 +1,5 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+export * from './slice';
+export * from './selectors';
+export { useGetWorkersQuery } from '../rollouts';

dashboard/src/features/workers/selectors.ts

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+import { createSelector } from '@reduxjs/toolkit';
+import type { GetWorkersQueryArgs } from '@/features/rollouts';
+import type { RootState } from '@/store';
+import type { WorkersSortState } from './slice';
+
+const WORKERS_SORT_FIELD_MAP: Record<string, string> = {
+  workerId: 'worker_id',
+  status: 'status',
+  currentRolloutId: 'current_rollout_id',
+  currentAttemptId: 'current_attempt_id',
+  lastHeartbeatTime: 'last_heartbeat_time',
+  lastDequeueTime: 'last_dequeue_time',
+  lastBusyTime: 'last_busy_time',
+  lastIdleTime: 'last_idle_time',
+};
+
+const resolveWorkersSortField = (sort: WorkersSortState): string =>
+  WORKERS_SORT_FIELD_MAP[sort.column] ?? 'last_heartbeat_time';
+
+export const selectWorkersUiState = (state: RootState) => state.workers;
+
+export const selectWorkersSearchTerm = (state: RootState) => selectWorkersUiState(state).searchTerm;
+export const selectWorkersPage = (state: RootState) => selectWorkersUiState(state).page;
+export const selectWorkersRecordsPerPage = (state: RootState) => selectWorkersUiState(state).recordsPerPage;
+export const selectWorkersSort = (state: RootState) => selectWorkersUiState(state).sort;
+
+export const selectWorkersQueryArgs = createSelector(
+  [selectWorkersSearchTerm, selectWorkersPage, selectWorkersRecordsPerPage, selectWorkersSort],
+  (searchTerm, page, recordsPerPage, sort): GetWorkersQueryArgs => {
+    const normalizedSearch = searchTerm.trim();
+    const limit = Math.max(1, recordsPerPage);
+    const offset = Math.max(0, (page - 1) * limit);
+    const sortBy = resolveWorkersSortField(sort);
+
+    return {
+      limit,
+      offset,
+      sortBy,
+      sortOrder: sort.direction,
+      workerIdContains: normalizedSearch.length > 0 ? normalizedSearch : undefined,
+    };
+  },
+);

dashboard/src/features/workers/slice.ts

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+import { createSlice, type PayloadAction } from '@reduxjs/toolkit';
+
+export type SortDirection = 'asc' | 'desc';
+
+export type WorkersSortState = {
+  column: string;
+  direction: SortDirection;
+};
+
+export type WorkersUiState = {
+  searchTerm: string;
+  page: number;
+  recordsPerPage: number;
+  sort: WorkersSortState;
+};
+
+export const initialWorkersUiState: WorkersUiState = {
+  searchTerm: '',
+  page: 1,
+  recordsPerPage: 50,
+  sort: {
+    column: 'lastHeartbeatTime',
+    direction: 'desc',
+  },
+};
+
+const workersSlice = createSlice({
+  name: 'workers',
+  initialState: initialWorkersUiState,
+  reducers: {
+    setWorkersSearchTerm(state, action: PayloadAction<string>) {
+      state.searchTerm = action.payload;
+      state.page = 1;
+    },
+    setWorkersPage(state, action: PayloadAction<number>) {
+      state.page = action.payload;
+    },
+    setWorkersRecordsPerPage(state, action: PayloadAction<number>) {
+      state.recordsPerPage = action.payload;
+      state.page = 1;
+    },
+    setWorkersSort(state, action: PayloadAction<WorkersSortState>) {
+      state.sort = action.payload;
+    },
+    resetWorkersFilters(state) {
+      state.searchTerm = initialWorkersUiState.searchTerm;
+      state.page = initialWorkersUiState.page;
+      state.recordsPerPage = initialWorkersUiState.recordsPerPage;
+      state.sort = initialWorkersUiState.sort;
+    },
+  },
+});
+
+export const { setWorkersSearchTerm, setWorkersPage, setWorkersRecordsPerPage, setWorkersSort, resetWorkersFilters } =
+  workersSlice.actions;
+
+export const workersReducer = workersSlice.reducer;

dashboard/src/features/workers/workers.test.ts

@@ -0,0 +1,102 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+import { createServerBackedStore } from '@test-utils';
+import { describe, expect, it } from 'vitest';
+import { rolloutsApi } from '@/features/rollouts';
+import type { Worker } from '@/types';
+import { selectWorkersQueryArgs } from './selectors';
+import {
+  resetWorkersFilters,
+  setWorkersPage,
+  setWorkersRecordsPerPage,
+  setWorkersSearchTerm,
+  setWorkersSort,
+} from './slice';
+
+const extractWorkerIds = (workers: Worker[]): string[] => workers.map((worker) => worker.workerId);
+
+describe('workers feature integration', () => {
+  it('builds default query arguments from the UI state', () => {
+    const store = createServerBackedStore();
+    const queryArgs = selectWorkersQueryArgs(store.getState());
+
+    expect(queryArgs).toMatchObject({
+      limit: 50,
+      offset: 0,
+      sortBy: 'last_heartbeat_time',
+      sortOrder: 'desc',
+      workerIdContains: undefined,
+    });
+  });
+
+  it('fetches workers from the Python LightningStore server', async () => {
+    const store = createServerBackedStore();
+    const queryArgs = selectWorkersQueryArgs(store.getState());
+
+    const subscription = store.dispatch(rolloutsApi.endpoints.getWorkers.initiate(queryArgs));
+    const data = await subscription.unwrap();
+    subscription.unsubscribe();
+
+    expect(data.total).toBeGreaterThanOrEqual(4);
+    expect(data.items).toHaveLength(Math.min(queryArgs.limit, data.total));
+
+    const workerIds = extractWorkerIds(data.items);
+    expect(workerIds).toEqual(expect.arrayContaining(['worker-east', 'worker-west']));
+
+    const heartbeatTimes = data.items.map((worker) => worker.lastHeartbeatTime ?? 0);
+    const sortedHeartbeatTimes = [...heartbeatTimes].sort((a, b) => b - a);
+    expect(heartbeatTimes).toEqual(sortedHeartbeatTimes);
+  });
+
+  it('paginates worker results based on UI state', async () => {
+    const store = createServerBackedStore();
+    store.dispatch(setWorkersRecordsPerPage(2));
+    store.dispatch(setWorkersPage(2));
+
+    const queryArgs = selectWorkersQueryArgs(store.getState());
+    expect(queryArgs).toMatchObject({ limit: 2, offset: 2 });
+
+    const subscription = store.dispatch(rolloutsApi.endpoints.getWorkers.initiate(queryArgs));
+    const data = await subscription.unwrap();
+    subscription.unsubscribe();
+
+    expect(data.items).toHaveLength(2);
+    expect(data.total).toBeGreaterThanOrEqual(4);
+  });
+
+  it('applies search and sorting preferences', async () => {
+    const store = createServerBackedStore();
+    store.dispatch(resetWorkersFilters());
+    store.dispatch(setWorkersSearchTerm('worker-west'));
+    store.dispatch(setWorkersSort({ column: 'workerId', direction: 'asc' }));
+
+    const queryArgs = selectWorkersQueryArgs(store.getState());
+    expect(queryArgs).toMatchObject({
+      limit: 50,
+      offset: 0,
+      sortBy: 'worker_id',
+      sortOrder: 'asc',
+      workerIdContains: 'worker-west',
+    });
+
+    const subscription = store.dispatch(rolloutsApi.endpoints.getWorkers.initiate(queryArgs));
+    const data = await subscription.unwrap();
+    subscription.unsubscribe();
+
+    expect(data.items).toHaveLength(1);
+    expect(data.items[0].workerId).toBe('worker-west');
+  });
+
+  it('maps current rollout/attempt sorting to backend fields', () => {
+    const store = createServerBackedStore();
+    store.dispatch(setWorkersSort({ column: 'currentRolloutId', direction: 'desc' }));
+    let queryArgs = selectWorkersQueryArgs(store.getState());
+    expect(queryArgs.sortBy).toBe('current_rollout_id');
+    expect(queryArgs.sortOrder).toBe('desc');
+
+    store.dispatch(setWorkersSort({ column: 'currentAttemptId', direction: 'asc' }));
+    queryArgs = selectWorkersQueryArgs(store.getState());
+    expect(queryArgs.sortBy).toBe('current_attempt_id');
+    expect(queryArgs.sortOrder).toBe('asc');
+  });
+});

dashboard/src/layouts/AppLayout.story.tsx

@@ -39,6 +39,10 @@ const ROUTES = [
         path: 'traces',
         element: <Placeholder title='Traces' description='Browse telemetry spans across attempts.' />,
       },
+      {
+        path: 'runners',
+        element: <Placeholder title='Runners' description='Monitor runner activity and status.' />,
+      },
       {
         path: 'settings',
         element: (

dashboard/src/layouts/AppLayout.tsx

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 import { useEffect, useMemo, useState, type ReactNode } from 'react';
-import { IconCpu, IconRouteSquare, IconSettings, IconTimeline } from '@tabler/icons-react';
+import { IconCpu, IconRouteSquare, IconRun, IconSettings, IconTimeline } from '@tabler/icons-react';
 import { Outlet, NavLink as RouterNavLink, useLocation, useNavigate } from 'react-router-dom';
 import { AppShell, Badge, Group, Image, NavLink as MantineNavLink, Stack, Text, UnstyledButton } from '@mantine/core';
 import { AppAlertBanner } from '@/components/AppAlertBanner';
@@ -22,6 +22,7 @@ const NAV_ITEMS: NavItem[] = [
   { label: 'Rollouts', to: '/rollouts', icon: <IconRouteSquare size={16} /> },
   { label: 'Resources', to: '/resources', icon: <IconCpu size={16} /> },
   { label: 'Traces', to: '/traces', icon: <IconTimeline size={16} /> },
+  { label: 'Runners', to: '/runners', icon: <IconRun size={16} /> },
   { label: 'Settings', to: '/settings', icon: <IconSettings size={16} /> },
 ];
 

dashboard/src/pages/Workers.page.story.tsx

@@ -0,0 +1,270 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+import type { Meta, StoryObj } from '@storybook/react';
+import { waitFor, within } from '@testing-library/dom';
+import userEvent from '@testing-library/user-event';
+import { Provider } from 'react-redux';
+import { createMemoryRouter, RouterProvider } from 'react-router-dom';
+import { AppAlertBanner } from '@/components/AppAlertBanner';
+import { AppDrawerContainer } from '@/components/AppDrawer.component';
+import { initialConfigState } from '@/features/config/slice';
+import { initialWorkersUiState } from '@/features/workers/slice';
+import { AppLayout } from '@/layouts/AppLayout';
+import { createAppStore } from '@/store';
+import type { Worker } from '@/types';
+import { createWorkersHandlers } from '@/utils/mock';
+import { STORY_BASE_URL, STORY_DATE_NOW_SECONDS } from '../../.storybook/constants';
+import { allModes } from '../../.storybook/modes';
+import { WorkersPage } from './Workers.page';
+
+const meta: Meta<typeof WorkersPage> = {
+  title: 'Pages/WorkersPage',
+  component: WorkersPage,
+  parameters: {
+    layout: 'fullscreen',
+    chromatic: {
+      modes: allModes,
+    },
+  },
+};
+
+export default meta;
+
+type Story = StoryObj<typeof WorkersPage>;
+
+const now = STORY_DATE_NOW_SECONDS;
+
+const sampleWorkers: Worker[] = [
+  {
+    workerId: 'worker-east',
+    status: 'busy',
+    heartbeatStats: { queueDepth: 2, gpuUtilization: 0.82 },
+    lastHeartbeatTime: now - 20,
+    lastDequeueTime: now - 120,
+    lastBusyTime: now - 60,
+    lastIdleTime: now - 600,
+    currentRolloutId: 'ro-story-001',
+    currentAttemptId: 'at-story-010',
+  },
+  {
+    workerId: 'worker-west',
+    status: 'busy',
+    heartbeatStats: { queueDepth: 1 },
+    lastHeartbeatTime: now - 45,
+    lastDequeueTime: now - 300,
+    lastBusyTime: now - 120,
+    lastIdleTime: now - 4800,
+    currentRolloutId: 'ro-story-003',
+    currentAttemptId: 'at-story-033',
+  },
+  {
+    workerId: 'worker-north',
+    status: 'idle',
+    heartbeatStats: { queueDepth: 0 },
+    lastHeartbeatTime: now - 120,
+    lastDequeueTime: now - 3600,
+    lastBusyTime: now - 5400,
+    lastIdleTime: now - 180,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+  {
+    workerId: 'worker-south',
+    status: 'idle',
+    heartbeatStats: null,
+    lastHeartbeatTime: now - 900,
+    lastDequeueTime: now - 7200,
+    lastBusyTime: now - 8600,
+    lastIdleTime: now - 8600,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+  {
+    workerId: 'worker-central',
+    status: 'busy',
+    heartbeatStats: { queueDepth: 3, cpuUtilization: 0.55 },
+    lastHeartbeatTime: now - 8,
+    lastDequeueTime: now - 45,
+    lastBusyTime: now - 10,
+    lastIdleTime: now - 900,
+    currentRolloutId: 'ro-story-005',
+    currentAttemptId: 'at-story-013',
+  },
+  {
+    workerId: 'worker-standby',
+    status: 'idle',
+    heartbeatStats: { queueDepth: 0, threads: 32 },
+    lastHeartbeatTime: now - 300,
+    lastDequeueTime: now - 10800,
+    lastBusyTime: now - 14400,
+    lastIdleTime: now - 200,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+  {
+    workerId: 'worker-observer',
+    status: 'unknown',
+    heartbeatStats: { queueDepth: 0 },
+    lastHeartbeatTime: now - 30,
+    lastDequeueTime: now - 6400,
+    lastBusyTime: null,
+    lastIdleTime: null,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+];
+
+const defaultHandlers = createWorkersHandlers(sampleWorkers);
+
+function createStoryStore(configOverrides?: Partial<typeof initialConfigState>) {
+  return createAppStore({
+    config: {
+      ...initialConfigState,
+      baseUrl: STORY_BASE_URL,
+      autoRefreshMs: 0,
+      ...configOverrides,
+    },
+    workers: initialWorkersUiState,
+  });
+}
+
+function renderWithStore(configOverrides?: Partial<typeof initialConfigState>) {
+  const store = createStoryStore(configOverrides);
+
+  return (
+    <Provider store={store}>
+      <>
+        <WorkersPage />
+        <AppAlertBanner />
+        <AppDrawerContainer />
+      </>
+    </Provider>
+  );
+}
+
+function renderWithinAppLayout(configOverrides?: Partial<typeof initialConfigState>) {
+  const store = createStoryStore(configOverrides);
+  const router = createMemoryRouter(
+    [
+      {
+        path: '/',
+        element: (
+          <AppLayout
+            config={{
+              baseUrl: store.getState().config.baseUrl,
+              autoRefreshMs: store.getState().config.autoRefreshMs,
+            }}
+          />
+        ),
+        children: [
+          {
+            path: '/runners',
+            element: <WorkersPage />,
+          },
+        ],
+      },
+    ],
+    { initialEntries: ['/runners'] },
+  );
+
+  return (
+    <Provider store={store}>
+      <>
+        <RouterProvider router={router} />
+        <AppDrawerContainer />
+      </>
+    </Provider>
+  );
+}
+
+const manyWorkers = Array.from({ length: 80 }, (_, index) => {
+  const suffix = (index + 1).toString().padStart(3, '0');
+  const busy = index % 2 === 0;
+  return {
+    workerId: `worker-batch-${suffix}`,
+    status: busy ? 'busy' : 'idle',
+    heartbeatStats: busy ? { queueDepth: (index % 5) + 1 } : { queueDepth: 0 },
+    lastHeartbeatTime: now - (index * 5 + 15),
+    lastDequeueTime: now - (index * 20 + 60),
+    lastBusyTime: busy ? now - (index * 10 + 30) : null,
+    lastIdleTime: busy ? null : now - (index * 10 + 45),
+    currentRolloutId: busy ? `ro-many-${suffix}` : null,
+    currentAttemptId: busy ? `at-many-${suffix}` : null,
+  } satisfies Worker;
+});
+
+export const Default: Story = {
+  render: () => renderWithinAppLayout(),
+  parameters: {
+    msw: {
+      handlers: defaultHandlers,
+    },
+  },
+};
+
+export const Search: Story = {
+  render: () => renderWithStore(),
+  parameters: {
+    msw: {
+      handlers: defaultHandlers,
+    },
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await canvas.findByText('worker-east');
+
+    const searchInput = canvas.getByPlaceholderText('Search by Runner ID');
+    await userEvent.type(searchInput, 'worker-west');
+
+    await waitFor(() => {
+      if (canvas.queryByText('worker-east')) {
+        throw new Error('Expected filtered table to hide worker-east');
+      }
+      if (!canvas.queryByText('worker-west')) {
+        throw new Error('Expected worker-west to remain visible');
+      }
+    });
+  },
+};
+
+export const DrawerOpen: Story = {
+  render: () => renderWithStore(),
+  parameters: {
+    msw: {
+      handlers: defaultHandlers,
+    },
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    await canvas.findByText('worker-east');
+
+    const detailsButtons = await canvas.findAllByRole('button', { name: /detail/i });
+    await userEvent.click(detailsButtons[0]);
+
+    const body = within(document.body);
+    await waitFor(() => {
+      if (!body.queryByTestId('json-editor-container')) {
+        throw new Error('Expected worker detail drawer with JSON view');
+      }
+    });
+  },
+};
+
+export const ManyWorkers: Story = {
+  render: () => renderWithStore(),
+  parameters: {
+    msw: {
+      handlers: createWorkersHandlers(manyWorkers),
+    },
+  },
+};
+
+export const DarkTheme: Story = {
+  render: () => renderWithStore({ theme: 'dark' }),
+  parameters: {
+    theme: 'dark',
+    msw: {
+      handlers: defaultHandlers,
+    },
+  },
+};

dashboard/src/pages/Workers.page.tsx

@@ -0,0 +1,159 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+import { useCallback, useEffect } from 'react';
+import { IconSearch } from '@tabler/icons-react';
+import type { DataTableSortStatus } from 'mantine-datatable';
+import { Skeleton, Stack, TextInput, Title } from '@mantine/core';
+import { WorkersTable, type WorkersTableRecord } from '@/components/WorkersTable.component';
+import { selectAutoRefreshMs } from '@/features/config';
+import { hideAlert, showAlert } from '@/features/ui/alert';
+import { openDrawer } from '@/features/ui/drawer';
+import {
+  resetWorkersFilters,
+  selectWorkersPage,
+  selectWorkersQueryArgs,
+  selectWorkersRecordsPerPage,
+  selectWorkersSearchTerm,
+  selectWorkersSort,
+  setWorkersPage,
+  setWorkersRecordsPerPage,
+  setWorkersSearchTerm,
+  setWorkersSort,
+  useGetWorkersQuery,
+} from '@/features/workers';
+import { useAppDispatch, useAppSelector } from '@/store/hooks';
+import type { PaginatedResponse, Worker } from '@/types';
+import { getErrorDescriptor } from '@/utils/error';
+
+export function WorkersPage() {
+  const dispatch = useAppDispatch();
+  const autoRefreshMs = useAppSelector(selectAutoRefreshMs);
+  const searchTerm = useAppSelector(selectWorkersSearchTerm);
+  const page = useAppSelector(selectWorkersPage);
+  const recordsPerPage = useAppSelector(selectWorkersRecordsPerPage);
+  const sort = useAppSelector(selectWorkersSort);
+  const queryArgs = useAppSelector(selectWorkersQueryArgs);
+
+  const workersQueryResult = useGetWorkersQuery(queryArgs, {
+    pollingInterval: autoRefreshMs > 0 ? autoRefreshMs : undefined,
+  });
+
+  const workersData = workersQueryResult.data as PaginatedResponse<Worker> | undefined;
+  const { isLoading, isFetching, isError, error, refetch } = workersQueryResult;
+
+  const handleSearchTermChange = useCallback(
+    (value: string) => {
+      dispatch(setWorkersSearchTerm(value));
+    },
+    [dispatch],
+  );
+
+  const handleSortStatusChange = useCallback(
+    (status: DataTableSortStatus<WorkersTableRecord>) => {
+      dispatch(
+        setWorkersSort({
+          column: status.columnAccessor,
+          direction: status.direction,
+        }),
+      );
+    },
+    [dispatch],
+  );
+
+  const handlePageChange = useCallback(
+    (nextPage: number) => {
+      dispatch(setWorkersPage(nextPage));
+    },
+    [dispatch],
+  );
+
+  const handleRecordsPerPageChange = useCallback(
+    (value: number) => {
+      dispatch(setWorkersRecordsPerPage(value));
+    },
+    [dispatch],
+  );
+
+  const handleResetFilters = useCallback(() => {
+    dispatch(resetWorkersFilters());
+  }, [dispatch]);
+
+  const handleShowWorkerDetails = useCallback(
+    (worker: Worker) => {
+      dispatch(
+        openDrawer({
+          type: 'worker-detail',
+          worker,
+        }),
+      );
+    },
+    [dispatch],
+  );
+
+  const hasWorkers = Array.isArray(workersData?.items) && workersData.items.length > 0;
+  const showSkeleton = isLoading && !hasWorkers;
+
+  useEffect(() => {
+    if (isError) {
+      const descriptor = getErrorDescriptor(error);
+      const suffix = descriptor ? ` (${descriptor})` : '';
+      dispatch(
+        showAlert({
+          id: 'workers-fetch',
+          message: `Unable to refresh workers${suffix}. The table may be out of date until the connection recovers.`,
+          tone: 'error',
+        }),
+      );
+      return;
+    }
+
+    if (!isLoading && !isFetching) {
+      dispatch(hideAlert({ id: 'workers-fetch' }));
+    }
+  }, [dispatch, error, isError, isFetching, isLoading]);
+
+  useEffect(
+    () => () => {
+      dispatch(hideAlert({ id: 'workers-fetch' }));
+    },
+    [dispatch],
+  );
+
+  return (
+    <Stack gap='md'>
+      <Title order={1}>Runners</Title>
+
+      <TextInput
+        placeholder='Search by Runner ID'
+        value={searchTerm}
+        onChange={(event) => handleSearchTermChange(event.currentTarget.value)}
+        leftSection={<IconSearch size={16} />}
+        data-testid='workers-search-input'
+        w='100%'
+        style={{ maxWidth: 360 }}
+      />
+
+      {showSkeleton ? (
+        <Skeleton height={360} radius='md' />
+      ) : (
+        <WorkersTable
+          workers={workersData?.items}
+          totalRecords={workersData?.total ?? 0}
+          isFetching={isFetching}
+          isError={isError}
+          error={error}
+          searchTerm={searchTerm}
+          sort={sort}
+          page={page}
+          recordsPerPage={recordsPerPage}
+          onSortStatusChange={handleSortStatusChange}
+          onPageChange={handlePageChange}
+          onRecordsPerPageChange={handleRecordsPerPageChange}
+          onResetFilters={handleResetFilters}
+          onRefetch={refetch}
+          onShowDetails={handleShowWorkerDetails}
+        />
+      )}
+    </Stack>
+  );
+}

dashboard/src/store/index.ts

@@ -7,6 +7,7 @@ import { rolloutsApi, rolloutsReducer } from '../features/rollouts';
 import { tracesReducer } from '../features/traces';
 import { alertReducer } from '../features/ui/alert';
 import { drawerReducer } from '../features/ui/drawer';
+import { workersReducer } from '../features/workers';
 
 const rootReducer = combineReducers({
   config: configReducer,
@@ -14,6 +15,7 @@ const rootReducer = combineReducers({
   alert: alertReducer,
   rollouts: rolloutsReducer,
   resources: resourcesReducer,
+  workers: workersReducer,
   traces: tracesReducer,
   [rolloutsApi.reducerPath]: rolloutsApi.reducer,
 });

dashboard/src/types.ts

@@ -28,6 +28,24 @@ export type Attempt = {
   metadata: Record<string, any> | null;
 };
 
+export type WorkerStatus = 'idle' | 'busy' | 'unknown';
+
+/**
+ * Synced with agentlightning.types.core.Worker
+ * with camel case and snake case conversions
+ */
+export type Worker = {
+  workerId: string;
+  status: WorkerStatus;
+  heartbeatStats: Record<string, any> | null;
+  lastHeartbeatTime: Timestamp | null;
+  lastDequeueTime: Timestamp | null;
+  lastBusyTime: Timestamp | null;
+  lastIdleTime: Timestamp | null;
+  currentRolloutId: string | null;
+  currentAttemptId: string | null;
+};
+
 /**
  * Synced with agentlightning.types.core.Rollout
  * with camel case and snake case conversions

dashboard/src/utils/mock.test.ts

@@ -8,27 +8,32 @@
  */
 
 import { describe, expect, it } from 'vitest';
-import type { Attempt, Resources, Rollout, Span } from '@/types';
+import type { Attempt, Resources, Rollout, Span, Worker } from '@/types';
 import {
   buildAttemptsResponse,
   buildResourcesResponse,
   buildRolloutsResponse,
   buildSpansResponse,
+  buildWorkersResponse,
   createMockHandlers,
   createResourcesHandlers,
   createRolloutsHandlers,
   createSpansHandlers,
+  createWorkersHandlers,
   filterResourcesForParams,
   filterRolloutsForParams,
   filterSpansForParams,
+  filterWorkersForParams,
   getResourcesSortValue,
   getRolloutSortValue,
   getSpanSortValue,
+  getWorkerSortValue,
   parseNumberParam,
   sortAttemptsForParams,
   sortResourcesForParams,
   sortRolloutsForParams,
   sortSpansForParams,
+  sortWorkersForParams,
 } from './mock';
 
 const now = Math.floor(Date.now() / 1000);
@@ -219,6 +224,53 @@ const sampleResources: Resources[] = [
   },
 ];
 
+const sampleWorkers: Worker[] = [
+  {
+    workerId: 'worker-alpha',
+    status: 'busy',
+    heartbeatStats: { queueDepth: 2 },
+    lastHeartbeatTime: now - 30,
+    lastDequeueTime: now - 300,
+    lastBusyTime: now - 60,
+    lastIdleTime: now - 600,
+    currentRolloutId: 'ro-001',
+    currentAttemptId: 'at-001',
+  },
+  {
+    workerId: 'worker-beta',
+    status: 'idle',
+    heartbeatStats: { queueDepth: 0 },
+    lastHeartbeatTime: now - 120,
+    lastDequeueTime: now - 1200,
+    lastBusyTime: now - 3600,
+    lastIdleTime: now - 180,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+  {
+    workerId: 'worker-gamma',
+    status: 'busy',
+    heartbeatStats: null,
+    lastHeartbeatTime: now - 10,
+    lastDequeueTime: now - 60,
+    lastBusyTime: now - 20,
+    lastIdleTime: now - 4000,
+    currentRolloutId: 'ro-003',
+    currentAttemptId: 'at-003',
+  },
+  {
+    workerId: 'worker-delta',
+    status: 'unknown',
+    heartbeatStats: { queueDepth: 0 },
+    lastHeartbeatTime: now - 5,
+    lastDequeueTime: now - 80,
+    lastBusyTime: null,
+    lastIdleTime: null,
+    currentRolloutId: null,
+    currentAttemptId: null,
+  },
+];
+
 describe('parseNumberParam', () => {
   it('returns default value when param is missing', () => {
     const params = new URLSearchParams();
@@ -725,6 +777,115 @@ describe('createResourcesHandlers', () => {
   });
 });
 
+describe('filterWorkersForParams', () => {
+  it('returns all workers without filters', () => {
+    const params = new URLSearchParams();
+    const result = filterWorkersForParams(sampleWorkers, params);
+    expect(result).toHaveLength(4);
+  });
+
+  it('filters by status and worker ID substring using AND logic', () => {
+    const params = new URLSearchParams('status_in=busy&worker_id_contains=gamma');
+    const result = filterWorkersForParams(sampleWorkers, params);
+    expect(result).toHaveLength(1);
+    expect(result[0].workerId).toBe('worker-gamma');
+  });
+
+  it('supports filter_logic=or', () => {
+    const params = new URLSearchParams('status_in=idle&worker_id_contains=gamma&filter_logic=or');
+    const result = filterWorkersForParams(sampleWorkers, params);
+    expect(result).toHaveLength(2);
+  });
+
+  it('filters by unknown status', () => {
+    const params = new URLSearchParams('status_in=unknown');
+    const result = filterWorkersForParams(sampleWorkers, params);
+    expect(result).toHaveLength(1);
+    expect(result[0].workerId).toBe('worker-delta');
+  });
+});
+
+describe('getWorkerSortValue', () => {
+  const worker = sampleWorkers[0];
+
+  it('returns worker_id', () => {
+    expect(getWorkerSortValue(worker, 'worker_id')).toBe('worker-alpha');
+  });
+
+  it('returns status', () => {
+    expect(getWorkerSortValue(worker, 'status')).toBe('busy');
+  });
+
+  it('returns timestamp fields', () => {
+    expect(getWorkerSortValue(worker, 'last_busy_time')).toBe(worker.lastBusyTime);
+    expect(getWorkerSortValue(worker, 'last_idle_time')).toBe(worker.lastIdleTime);
+    expect(getWorkerSortValue(worker, 'last_dequeue_time')).toBe(worker.lastDequeueTime);
+  });
+
+  it('returns rollout and attempt identifiers', () => {
+    expect(getWorkerSortValue(worker, 'current_rollout_id')).toBe(worker.currentRolloutId);
+    expect(getWorkerSortValue(worker, 'current_attempt_id')).toBe(worker.currentAttemptId);
+  });
+
+  it('falls back to last_heartbeat_time', () => {
+    expect(getWorkerSortValue(worker, 'unknown')).toBe(worker.lastHeartbeatTime);
+  });
+});
+
+describe('sortWorkersForParams', () => {
+  it('sorts by last heartbeat ascending by default', () => {
+    const result = sortWorkersForParams(sampleWorkers, null, 'asc');
+    expect(result.map((worker) => worker.workerId)).toEqual([
+      'worker-beta',
+      'worker-alpha',
+      'worker-gamma',
+      'worker-delta',
+    ]);
+  });
+
+  it('sorts descending by worker_id when requested', () => {
+    const result = sortWorkersForParams(sampleWorkers, 'worker_id', 'desc');
+    expect(result.map((worker) => worker.workerId)).toEqual([
+      'worker-gamma',
+      'worker-delta',
+      'worker-beta',
+      'worker-alpha',
+    ]);
+  });
+
+  it('sorts by current_rollout_id', () => {
+    const result = sortWorkersForParams(sampleWorkers, 'current_rollout_id', 'asc');
+    expect(result.map((worker) => worker.currentRolloutId)).toEqual([null, null, 'ro-001', 'ro-003']);
+  });
+});
+
+describe('buildWorkersResponse', () => {
+  it('applies filters before pagination', () => {
+    const request = new Request('http://localhost/v1/agl/workers?worker_id_contains=beta&limit=5');
+    const response = buildWorkersResponse(sampleWorkers, request);
+    expect(response.items).toHaveLength(1);
+    const items = response.items as Array<Record<string, unknown>>;
+    expect(items[0].worker_id).toBe('worker-beta');
+  });
+
+  it('applies sort and pagination parameters', () => {
+    const request = new Request('http://localhost/v1/agl/workers?sort_by=worker_id&limit=2&offset=1');
+    const response = buildWorkersResponse(sampleWorkers, request);
+    expect(response.items).toHaveLength(2);
+    const items = response.items as Array<Record<string, unknown>>;
+    expect(items[0].worker_id).toBe('worker-beta');
+    expect(response.total).toBe(4);
+  });
+});
+
+describe('createWorkersHandlers', () => {
+  it('creates handler for workers endpoint', () => {
+    const handlers = createWorkersHandlers(sampleWorkers);
+    expect(handlers).toHaveLength(1);
+    expect(handlers[0].info.header).toContain('GET');
+  });
+});
+
 describe('createRolloutsHandlers', () => {
   it('creates handlers that return correct rollout data', async () => {
     const attemptsByRollout = { 'ro-001': sampleAttempts };

dashboard/src/utils/mock.ts

@@ -14,7 +14,7 @@
  */
 
 import { delay, http, HttpResponse } from 'msw';
-import type { Attempt, Resources, Rollout, Span } from '@/types';
+import type { Attempt, Resources, Rollout, Span, Worker } from '@/types';
 import { snakeCaseKeys } from './format';
 
 /**
@@ -434,6 +434,117 @@ export function buildResourcesResponse(resources: Resources[], request: Request)
   });
 }
 
+/**
+ * Filter workers based on query parameters.
+ * Supports: status_in, worker_id_contains
+ */
+export function filterWorkersForParams(workers: Worker[], params: URLSearchParams): Worker[] {
+  const statusFilters = params.getAll('status_in');
+  const workerIdContains = params.get('worker_id_contains');
+  const filterLogic = params.get('filter_logic') === 'or' ? 'or' : 'and';
+
+  return workers.filter((worker) => {
+    const checks: boolean[] = [];
+    if (statusFilters.length > 0) {
+      checks.push(statusFilters.includes(worker.status));
+    }
+    if (workerIdContains) {
+      checks.push(worker.workerId.toLowerCase().includes(workerIdContains.toLowerCase()));
+    }
+    if (checks.length === 0) {
+      return true;
+    }
+    return filterLogic === 'or' ? checks.some(Boolean) : checks.every(Boolean);
+  });
+}
+
+/**
+ * Resolve a worker sort value for the given column.
+ */
+export function getWorkerSortValue(worker: Worker, sortBy: string): string | number | null {
+  switch (sortBy) {
+    case 'worker_id':
+      return worker.workerId;
+    case 'status':
+      return worker.status;
+    case 'current_rollout_id':
+      return worker.currentRolloutId ?? '';
+    case 'current_attempt_id':
+      return worker.currentAttemptId ?? '';
+    case 'last_busy_time':
+      return worker.lastBusyTime ?? null;
+    case 'last_idle_time':
+      return worker.lastIdleTime ?? null;
+    case 'last_dequeue_time':
+      return worker.lastDequeueTime ?? null;
+    case 'last_heartbeat_time':
+    default:
+      return worker.lastHeartbeatTime ?? null;
+  }
+}
+
+/**
+ * Sort workers based on query parameters.
+ * Default sort_by is 'last_heartbeat_time'.
+ */
+export function sortWorkersForParams(workers: Worker[], sortBy: string | null, sortOrder: 'asc' | 'desc'): Worker[] {
+  const resolvedSortBy = sortBy ?? 'last_heartbeat_time';
+  const sorted = [...workers].sort((a, b) => {
+    const aValue = getWorkerSortValue(a, resolvedSortBy);
+    const bValue = getWorkerSortValue(b, resolvedSortBy);
+    if (aValue === bValue) {
+      return 0;
+    }
+    if (aValue == null) {
+      return -1;
+    }
+    if (bValue == null) {
+      return 1;
+    }
+    if (typeof aValue === 'number' && typeof bValue === 'number') {
+      return aValue - bValue;
+    }
+    return String(aValue).localeCompare(String(bValue));
+  });
+
+  if (sortOrder === 'desc') {
+    sorted.reverse();
+  }
+
+  return sorted;
+}
+
+/**
+ * Build a paginated workers response matching the Python server's format.
+ */
+export function buildWorkersResponse(workers: Worker[], request: Request): Record<string, unknown> {
+  const url = new URL(request.url);
+  const params = url.searchParams;
+  const filtered = filterWorkersForParams(workers, params);
+  const sortBy = params.get('sort_by');
+  const sortOrder = params.get('sort_order') === 'desc' ? 'desc' : 'asc';
+  const sorted = sortWorkersForParams(filtered, sortBy, sortOrder);
+  const limitParam = parseNumberParam(params, 'limit', sorted.length);
+  const offsetParam = parseNumberParam(params, 'offset', 0);
+  const effectiveLimit = limitParam < 0 ? sorted.length : limitParam;
+  const offset = offsetParam < 0 ? 0 : offsetParam;
+  const paginated = effectiveLimit >= 0 ? sorted.slice(offset, offset + effectiveLimit) : [...sorted];
+
+  return snakeCaseKeys({
+    items: paginated,
+    limit: effectiveLimit,
+    offset,
+    total: filtered.length,
+  });
+}
+
+/**
+ * Create MSW handlers for workers endpoints.
+ */
+export function createWorkersHandlers(workers: Worker[]) {
+  return [http.get('*/v1/agl/workers', ({ request }) => HttpResponse.json(buildWorkersResponse(workers, request)))];
+}
+
 /**
  * Create MSW handlers for resources endpoints.
  *

dashboard/test-utils/python-server.py

@@ -33,10 +33,11 @@ from agentlightning.types import (
     RolloutConfig,
     Span,
     TraceStatus,
+    Worker,
 )
 
 
-def inject_mock_data(store: InMemoryLightningStore, now: float | None = None) -> None:
+async def inject_mock_data(store: InMemoryLightningStore, now: float | None = None) -> None:
     """
     Inject mock data directly into the InMemoryLightningStore.
 
@@ -216,20 +217,12 @@ def inject_mock_data(store: InMemoryLightningStore, now: float | None = None) ->
     )
 
     # Inject rollouts directly into store
-    store._rollouts["ro-story-001"] = rollout1
-    store._rollouts["ro-story-002"] = rollout2
-    store._rollouts["ro-story-003"] = rollout3
-    store._rollouts["ro-story-004"] = rollout4
-    store._rollouts["ro-story-005"] = rollout5
-    store._rollouts["ro-story-006"] = rollout6
+    await store.collections.rollouts.insert([rollout1, rollout2, rollout3, rollout4, rollout5, rollout6])
 
     # Inject attempts directly into store
-    store._attempts["ro-story-001"] = [attempt1]
-    store._attempts["ro-story-002"] = [attempt2_1, attempt2_2]
-    store._attempts["ro-story-003"] = [attempt3_1, attempt3_2, attempt3_3]
-    store._attempts["ro-story-004"] = []  # No attempt for preparing rollout
-    store._attempts["ro-story-005"] = [attempt5]
-    store._attempts["ro-story-006"] = [attempt6]
+    await store.collections.attempts.insert(
+        [attempt1, attempt2_1, attempt2_2, attempt3_1, attempt3_2, attempt3_3, attempt5, attempt6]
+    )
 
     # Create and inject spans with diverse data
     # Spans for ro-story-001 (Running) - Multiple nested spans with ongoing execution
@@ -544,11 +537,7 @@ def inject_mock_data(store: InMemoryLightningStore, now: float | None = None) ->
         ),
     ]
 
-    store._spans["ro-story-001"] = spans_ro1
-    store._spans["ro-story-002"] = spans_ro2_a1 + spans_ro2_a2
-    store._spans["ro-story-003"] = spans_ro3_a3
-    store._spans["ro-story-005"] = spans_ro5
-    store._spans["ro-story-006"] = spans_ro6
+    await store.collections.spans.insert(spans_ro1 + spans_ro2_a1 + spans_ro2_a2 + spans_ro3_a3 + spans_ro5 + spans_ro6)
 
     # Create and inject resources with diverse types
     resource1 = ResourcesUpdate(
@@ -627,13 +616,70 @@ def inject_mock_data(store: InMemoryLightningStore, now: float | None = None) ->
         },
     )
 
-    store._resources["rs-story-001"] = resource1
-    store._resources["rs-story-002"] = resource2
-    store._resources["rs-story-003"] = resource3
-    store._resources["rs-story-004"] = resource4
-    store._resources["rs-story-005"] = resource5
+    await store.collections.resources.insert([resource1, resource2, resource3, resource4, resource5])
     store._latest_resources_id = "rs-story-005"
 
+    # Register workers with diverse states and activity windows.
+    workers = [
+        Worker(
+            worker_id="worker-east",
+            status="busy",
+            heartbeat_stats={"queue_depth": 2, "gpu_utilization": 0.82},
+            last_heartbeat_time=now - 20,
+            last_dequeue_time=now - 60,
+            last_busy_time=now - 120,
+            last_idle_time=now - 600,
+            current_rollout_id="ro-story-001",
+            current_attempt_id="at-story-010",
+        ),
+        Worker(
+            worker_id="worker-north",
+            status="idle",
+            heartbeat_stats={"queue_depth": 0, "gpu_utilization": 0.15},
+            last_heartbeat_time=now - 90,
+            last_dequeue_time=now - 3600,
+            last_busy_time=now - 5400,
+            last_idle_time=now - 5400,
+            current_rollout_id=None,
+            current_attempt_id=None,
+        ),
+        Worker(
+            worker_id="worker-west",
+            status="busy",
+            heartbeat_stats={"queue_depth": 1, "gpu_utilization": 0.41},
+            last_heartbeat_time=now - 45,
+            last_dequeue_time=now - 300,
+            last_busy_time=now - 200,
+            last_idle_time=now - 4800,
+            current_rollout_id="ro-story-003",
+            current_attempt_id="at-story-033",
+        ),
+        Worker(
+            worker_id="worker-south",
+            status="idle",
+            heartbeat_stats={"queue_depth": 0},
+            last_heartbeat_time=now - 900,
+            last_dequeue_time=now - 7200,
+            last_busy_time=now - 8600,
+            last_idle_time=now - 8600,
+            current_rollout_id=None,
+            current_attempt_id=None,
+        ),
+        Worker(
+            worker_id="worker-observer",
+            status="unknown",
+            heartbeat_stats={"queue_depth": 0},
+            last_heartbeat_time=now - 15,
+            last_dequeue_time=now - 4000,
+            last_busy_time=None,
+            last_idle_time=None,
+            current_rollout_id=None,
+            current_attempt_id=None,
+        ),
+    ]
+
+    await store.collections.workers.insert(workers)
+
 
 async def main():
     parser = argparse.ArgumentParser(description="Run a Python server for the LightningStore")
@@ -641,7 +687,7 @@ async def main():
     args = parser.parse_args()
 
     store = InMemoryLightningStore()
-    inject_mock_data(store, now=args.now)
+    await inject_mock_data(store, now=args.now)
 
     # Start server
     server = LightningStoreServer(store, "127.0.0.1", 8765, "*")

docker/Dockerfile.dev

@@ -0,0 +1,39 @@
+# Use a Python image with uv pre-installed
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm
+
+# Setup a non-root user
+RUN groupadd --system --gid 999 nonroot \
+ && useradd --system --gid 999 --uid 999 --create-home nonroot
+
+# Install the project into `/app`
+WORKDIR /app
+
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+# Ensure installed tools can be executed out of the box
+ENV UV_TOOL_BIN_DIR=/usr/local/bin
+
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --locked --no-install-project --group dev --extra mongo --group core-stable
+
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+COPY . /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --group dev --extra mongo --group core-stable
+
+# Place executables in the environment at the front of the path
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Reset the entrypoint, don't invoke `uv`
+ENTRYPOINT []
+
+# Use the non-root user to run our application
+USER nonroot

docker/compose.grafana.yml

@@ -0,0 +1,29 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+    volumes:
+      - ./prometheus.memory-store.yml:/etc/prometheus/prometheus.yml:ro
+      - ${AGL_MONITORING_DATA_PATH:?Set AGL_MONITORING_DATA_PATH to the metrics directory}/prometheus:/prometheus
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana:latest
+    depends_on:
+      - prometheus
+    ports:
+      - "9091:3000"
+    volumes:
+      - ./data/grafana:/var/lib/grafana
+      - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./grafana/dashboard-provider.yml:/etc/grafana/provisioning/dashboards/provider.yml
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+    environment:
+      - GF_INSTALL_PLUGINS=grafana-piechart-panel
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/agentlightning.json

docker/compose.mongo.yml

@@ -0,0 +1,22 @@
+# Docker-compose file to launch a MongoDB server for development.
+# It's used to test the MongoDB store implementation.
+
+services:
+  mongo:
+    image: mongo:8.2
+    ulimits:
+      nofile:
+        soft: 65535
+        hard: 65535
+    ports:
+      - "27017:27017"
+    command: ["mongod", "--bind_ip_all", "--replSet", "rs0"]
+    volumes:
+      - ../scripts/mongodb_init_rs_host.js:/docker-entrypoint-initdb.d/init-rs.js:ro
+      - ./data/mongo-host:/data/db
+    healthcheck:
+      test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s

docker/compose.prometheus-memory-store.yml

@@ -0,0 +1,53 @@
+services:
+  app:
+    extends:
+      file: compose.store.yml
+      service: app
+
+    command: agl store --host 0.0.0.0 --port 4747 --prometheus --backend memory
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    # In CI you might not have full /proc, but this is OK for container-level stats
+    pid: "host"
+    command:
+      - "--path.rootfs=/host"
+    volumes:
+      - "/:/host:ro,rslave"
+
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=1h"
+    volumes:
+      - ./prometheus.memory-store.yml:/etc/prometheus/prometheus.yml:ro
+      - ./data/prometheus:/prometheus
+    depends_on:
+      - app
+      - node-exporter
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "9091:3000"
+    depends_on:
+      - prometheus
+    volumes:
+      - ./data/grafana:/var/lib/grafana
+      # 1. Mount the Datasource Config
+      - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      # 2. Mount the Dashboard Provider Config
+      - ./grafana/dashboard-provider.yml:/etc/grafana/provisioning/dashboards/provider.yml
+      # 3. Mount the folder containing the actual JSON files
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+
+    environment:
+      - GF_INSTALL_PLUGINS=grafana-piechart-panel
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/agentlightning.json