.

.github/workflows/benchmark.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   benchmark:
-    name: ${{ matrix.workload.kind }} (${{ matrix.backend.id }}, ${{ matrix.workload.display }})
+    name: ${{ matrix.workload.kind }} (${{ matrix.backend.id }}, ${{ matrix.workload.display }}, ${{ matrix.trace_sink }})
     runs-on: ${{ matrix.workload.runner }}
     timeout-minutes: ${{ matrix.workload.timeout }}
     strategy:
@@ -20,67 +20,69 @@ jobs:
             compose_file: compose.prometheus-memory-store.yml
           - id: mongo
             compose_file: compose.prometheus-mongo-store.yml
+        trace_sink: [store, kafka, clickhouse]
+        # trace_sink: [clickhouse]
         workload:
-          - id: scenario-minimal-scale
-            display: Minimal production scale
-            kind: scenario
-            store_workers: 4
-            runner:
-              - self-hosted
-              - 1ES.Pool=agl-runner-cpu
-            timeout: 45
-            args: >-
-              --mode batch
-              --total-tasks 4096
-              --batch-size 256
-              --n-runners 32
-              --max-rounds 6
-              --sleep-seconds 0.5
-          - id: scenario-medium-scale
-            display: Medium production scale
-            kind: scenario
-            store_workers: 16
-            runner:
-              - self-hosted
-              - 1ES.Pool=agl-runner-cpu
-            timeout: 45
-            args: >-
-              --mode batch
-              --total-tasks 10000
-              --batch-size 1000
-              --n-runners 100
-              --max-rounds 10
-              --sleep-seconds 0.1
-          - id: scenario-midhigh-scale
-            display: Mid-high production scale
-            kind: scenario
-            store_workers: 24
-            runner:
-              - self-hosted
-              - 1ES.Pool=agl-runner-cpu
-            timeout: 60
-            args: >-
-              --mode batch
-              --total-tasks 20000
-              --batch-size 2048
-              --n-runners 256
-              --max-rounds 8
-              --sleep-seconds 0.1
-          - id: scenario-large-batch
-            display: Large batch waves
-            kind: scenario
-            store_workers: 64
-            runner:
-              - self-hosted
-              - 1ES.Pool=agl-runner-cpu-high
-            timeout: 120
-            args: >-
-              --mode batch
-              --total-tasks 50000
-              --batch-size 8192
-              --n-runners 256
-              --max-rounds 6
-              --sleep-seconds 0.1
+          # - id: scenario-minimal-scale
+          #   display: Minimal production scale
+          #   kind: scenario
+          #   store_workers: 4
+          #   runner:
+          #     - self-hosted
+          #     - 1ES.Pool=agl-runner-cpu
+          #   timeout: 45
+          #   args: >-
+          #     --mode batch
+          #     --total-tasks 4096
+          #     --batch-size 256
+          #     --n-runners 32
+          #     --max-rounds 6
+          #     --sleep-seconds 0.5
+          # - id: scenario-medium-scale
+          #   display: Medium production scale
+          #   kind: scenario
+          #   store_workers: 16
+          #   runner:
+          #     - self-hosted
+          #     - 1ES.Pool=agl-runner-cpu
+          #   timeout: 45
+          #   args: >-
+          #     --mode batch
+          #     --total-tasks 10000
+          #     --batch-size 1000
+          #     --n-runners 100
+          #     --max-rounds 10
+          #     --sleep-seconds 0.1
+          # - id: scenario-midhigh-scale
+          #   display: Mid-high production scale
+          #   kind: scenario
+          #   store_workers: 24
+          #   runner:
+          #     - self-hosted
+          #     - 1ES.Pool=agl-runner-cpu
+          #   timeout: 60
+          #   args: >-
+          #     --mode batch
+          #     --total-tasks 20000
+          #     --batch-size 2048
+          #     --n-runners 300
+          #     --max-rounds 6
+          #     --sleep-seconds 0.1
+          # - id: scenario-large-batch
+          #   display: Large batch waves
+          #   kind: scenario
+          #   store_workers: 96
+          #   runner:
+          #     - self-hosted
+          #     - 1ES.Pool=agl-runner-cpu-high
+          #   timeout: 120
+          #   args: >-
+          #     --mode batch
+          #     --total-tasks 50000
+          #     --batch-size 8192
+          #     --n-runners 1000
+          #     --max-rounds 3
+          #     --sleep-seconds 0.1
           - id: scenario-long-queues
             display: Long rollout queues
             kind: scenario
@@ -97,25 +99,25 @@ jobs:
               --remaining-tasks 4096
               --max-rounds 4
               --sleep-seconds 0.1
-          - id: scenario-high-concurrency
-            display: High-throughput concurrent requests
-            kind: scenario
-            store_workers: 96
-            runner:
-              - self-hosted
-              - 1ES.Pool=agl-runner-cpu-high
-            timeout: 120
-            args: >-
-              --mode single
-              --total-tasks 50000
-              --concurrency 2048
-              --n-runners 256
-              --max-rounds 2
-              --sleep-seconds 0.1
+          # - id: scenario-high-concurrency
+          #   display: High-throughput concurrent requests
+          #   kind: scenario
+          #   store_workers: 96
+          #   runner:
+          #     - self-hosted
+          #     - 1ES.Pool=agl-runner-cpu-high
+          #   timeout: 120
+          #   args: >-
+          #     --mode single
+          #     --total-tasks 50000
+          #     --concurrency 2048
+          #     --n-runners 256
+          #     --max-rounds 2
+          #     --sleep-seconds 0.1
           - id: scenario-heavy-traces
             display: Heavy rollouts with deep traces
             kind: scenario
-            store_workers: 64
+            store_workers: 96
             runner:
               - self-hosted
               - 1ES.Pool=agl-runner-cpu-high
@@ -129,48 +131,48 @@ jobs:
               --max-rounds 20
               --sleep-seconds 1.0
 
-          - id: micro-worker
-            display: Update worker
-            kind: micro
-            store_workers: 8
-            runner: ubuntu-latest
-            timeout: 30
-            cli: worker
-          - id: micro-dequeue-empty
-            display: Dequeue empty
-            kind: micro
-            store_workers: 8
-            runner: ubuntu-latest
-            timeout: 30
-            cli: dequeue-empty
-          - id: micro-rollout
-            display: Rollout + span
-            kind: micro
-            store_workers: 8
-            runner: ubuntu-latest
-            timeout: 30
-            cli: rollout
-          - id: micro-dequeue-update-attempt
-            display: Dequeue + update attempt
-            kind: micro
-            store_workers: 8
-            runner: ubuntu-latest
-            timeout: 30
-            cli: dequeue-update-attempt
-          - id: micro-dequeue-only
-            display: Dequeue only
-            kind: micro
-            store_workers: 8
-            runner: ubuntu-latest
-            timeout: 30
-            cli: dequeue-only
-          - id: micro-metrics
-            display: Multi-metric fan-out
-            kind: micro
-            store_workers: 8
-            runner: ubuntu-latest
-            timeout: 15
-            cli: metrics
+          # - id: micro-worker
+          #   display: Update worker
+          #   kind: micro
+          #   store_workers: 8
+          #   runner: ubuntu-latest
+          #   timeout: 30
+          #   cli: worker
+          # - id: micro-dequeue-empty
+          #   display: Dequeue empty
+          #   kind: micro
+          #   store_workers: 8
+          #   runner: ubuntu-latest
+          #   timeout: 30
+          #   cli: dequeue-empty
+          # - id: micro-rollout
+          #   display: Rollout + span
+          #   kind: micro
+          #   store_workers: 8
+          #   runner: ubuntu-latest
+          #   timeout: 30
+          #   cli: rollout
+          # - id: micro-dequeue-update-attempt
+          #   display: Dequeue + update attempt
+          #   kind: micro
+          #   store_workers: 8
+          #   runner: ubuntu-latest
+          #   timeout: 30
+          #   cli: dequeue-update-attempt
+          # - id: micro-dequeue-only
+          #   display: Dequeue only
+          #   kind: micro
+          #   store_workers: 8
+          #   runner: ubuntu-latest
+          #   timeout: 30
+          #   cli: dequeue-only
+          # - id: micro-metrics
+          #   display: Multi-metric fan-out
+          #   kind: micro
+          #   store_workers: 8
+          #   runner: ubuntu-latest
+          #   timeout: 15
+          #   cli: metrics
     env:
       PYTHONUNBUFFERED: "1"
       STORE_URL: http://localhost:4747
@@ -180,13 +182,14 @@ jobs:
       WORKLOAD_KIND: ${{ matrix.workload.kind }}
       WORKLOAD_ID: ${{ matrix.workload.id }}
       BACKEND_ID: ${{ matrix.backend.id }}
-      ARTIFACT_DIR: ${{ format('artifacts/{0}-{1}', matrix.workload.id, matrix.backend.id) }}
+      TRACE_SINK_ID: ${{ matrix.trace_sink }}
+      ARTIFACT_DIR: ${{ format('artifacts/{0}-{1}-{2}', matrix.workload.id, matrix.backend.id, matrix.trace_sink) }}
       COMPOSE_FILE: ${{ matrix.backend.compose_file }}
       AGL_STORE_N_WORKERS: ${{ matrix.workload.store_workers }}
-      ANALYSIS_FILE: ${{ format('analysis-{0}.log', matrix.workload.id) }}
-      SUMMARY_FILE: ${{ format('summary-{0}.log', matrix.workload.id) }}
+      ANALYSIS_FILE: ${{ format('analysis-{0}-{1}.log', matrix.workload.id, matrix.trace_sink) }}
+      SUMMARY_FILE: ${{ format('summary-{0}-{1}.log', matrix.workload.id, matrix.trace_sink) }}
       PROM_ARCHIVE_BASENAME: ${{ format('prometheus-{0}-{1}', matrix.workload.id, matrix.backend.id) }}
-      ARTIFACT_NAME: ${{ format('{0}-{1}', matrix.workload.id, matrix.backend.id) }}
+      ARTIFACT_NAME: ${{ format('{0}-{1}-{2}', matrix.workload.id, matrix.backend.id, matrix.trace_sink) }}
     steps:
       - uses: actions/checkout@v4
 
@@ -232,9 +235,217 @@ jobs:
           cd docker && docker compose -f "$COMPOSE_FILE" logs app
           exit 1
 
+
+      - name: Configure trace sink (store vs otlp)
+        run: |
+          set -euo pipefail
+          if [ "${{ matrix.trace_sink }}" = "kafka" ] || [ "${{ matrix.trace_sink }}" = "clickhouse" ]; then
+            echo "AGL_OTLP_ENDPOINT=http://localhost:4318/v1/traces" >> "$GITHUB_ENV"
+          fi
+
+      - name: Launch Kafka + OTel Collector (OTLP -> Kafka)
+        if: ${{ matrix.trace_sink == 'kafka' }}
+        run: |
+          set -euo pipefail
+          cd docker
+
+          # Generate OTel Collector config (OTLP/HTTP receiver -> Kafka exporter)
+          cat > otelcol-kafka.yml <<'YAML'
+          receivers:
+            otlp:
+              protocols:
+                http:
+                  endpoint: 0.0.0.0:4318
+          processors:
+            batch: {}
+          exporters:
+            kafka:
+              brokers: ["kafka:9092"]
+              topic: "agl-otlp-spans"
+              encoding: otlp_proto
+          service:
+            pipelines:
+              traces:
+                receivers: [otlp]
+                processors: [batch]
+                exporters: [kafka]
+          YAML
+
+          # Launch Kafka + Zookeeper + OTel Collector
+          cat > compose.kafka-otel.yml <<'YAML'
+          services:
+            zookeeper:
+              image: confluentinc/cp-zookeeper:7.6.1
+              environment:
+                ZOOKEEPER_CLIENT_PORT: 2181
+                ZOOKEEPER_TICK_TIME: 2000
+
+            kafka:
+              image: confluentinc/cp-kafka:7.6.1
+              depends_on: [zookeeper]
+              environment:
+                KAFKA_BROKER_ID: 1
+                KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+                KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+                KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+                KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+                KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+                # Enlarge max message size to accommodate large spans
+                KAFKA_MESSAGE_MAX_BYTES: "10000000"
+                KAFKA_REPLICA_FETCH_MAX_BYTES: "10000000"
+                KAFKA_SOCKET_REQUEST_MAX_BYTES: "10000000"
+
+            otelcol:
+              image: otel/opentelemetry-collector-contrib:latest
+              depends_on: [kafka]
+              command: ["--config=/etc/otelcol/config.yml"]
+              # command:
+              #   - "--config=/etc/otelcol/config.yml"
+              #   - "--set=service.telemetry.logs.level=debug"
+              volumes:
+                - ./otelcol-kafka.yml:/etc/otelcol/config.yml:ro
+              ports:
+                - "4318:4318"
+          YAML
+
+          docker compose -p agl-kafka -f compose.kafka-otel.yml down -v || true
+          docker compose -p agl-kafka -f compose.kafka-otel.yml up -d --quiet-pull
+
+          # Create topic (idempotent)
+          docker compose -p agl-kafka -f compose.kafka-otel.yml exec -T kafka \
+            kafka-topics --bootstrap-server kafka:9092 \
+            --create --if-not-exists \
+            --topic agl-otlp-spans --partitions 3 --replication-factor 1
+
+          # Wait for OTLP/HTTP port to be reachable on the host
+          for attempt in {1..30}; do
+            if (echo > /dev/tcp/127.0.0.1/4318) >/dev/null 2>&1; then
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "OTel Collector port 4318 not reachable in time" >&2
+          docker compose -p agl-kafka -f compose.kafka-otel.yml logs otelcol || true
+          exit 1
+
+      - name: Launch ClickHouse + OTel Collector (OTLP -> ClickHouse)
+        if: ${{ matrix.trace_sink == 'clickhouse' }}
+        run: |
+          set -euo pipefail
+          cd docker
+
+          # Generate OTel Collector config (OTLP/HTTP receiver -> ClickHouse exporter)
+          cat > otelcol-clickhouse.yml <<'YAML'
+          receivers:
+            otlp:
+              protocols:
+                http:
+                  endpoint: 0.0.0.0:4318
+          processors:
+            batch: {}
+          exporters:
+            clickhouse:
+              endpoint: tcp://clickhouse:9000?dial_timeout=10s&compress=lz4
+              database: otel
+              traces_table_name: otel_traces
+              username: otel
+              password: ${env:CLICKHOUSE_PASSWORD}
+          service:
+            pipelines:
+              traces:
+                receivers: [otlp]
+                processors: [batch]
+                exporters: [clickhouse]
+          YAML
+
+          # Launch ClickHouse + OTel Collector
+          cat > compose.clickhouse-otel.yml <<'YAML'
+          services:
+            clickhouse:
+              image: clickhouse/clickhouse-server:latest
+              environment:
+                CLICKHOUSE_USER: "otel"
+                CLICKHOUSE_PASSWORD: "changeme"
+                CLICKHOUSE_DB: "otel"
+              ulimits:
+                nofile:
+                  soft: 262144
+                  hard: 262144
+              ports:
+                - "8123:8123"
+                - "9000:9000"
+              volumes:
+                - ch_data:/var/lib/clickhouse
+              healthcheck:
+                test: ["CMD-SHELL", "wget -qO- http://localhost:8123/ping | grep -q Ok"]
+                interval: 2s
+                timeout: 2s
+                retries: 30
+                start_period: 10s
+
+            otelcol:
+              image: otel/opentelemetry-collector-contrib:latest
+              depends_on:
+                clickhouse:
+                  condition: service_healthy
+              restart: unless-stopped
+              command:
+                - "--config=/etc/otelcol/config.yml"
+                - "--set=service.telemetry.logs.level=debug"
+              environment:
+                CLICKHOUSE_PASSWORD: "changeme"
+              volumes:
+                - ./otelcol-clickhouse.yml:/etc/otelcol/config.yml:ro
+              ports:
+                - "4318:4318"
+
+          volumes:
+            ch_data:
+
+          YAML
+
+          docker compose -p agl-clickhouse -f compose.clickhouse-otel.yml down -v || true
+          docker compose -p agl-clickhouse -f compose.clickhouse-otel.yml up -d --quiet-pull
+
+          # Wait for OTLP/HTTP port to be reachable on the host
+          for attempt in {1..30}; do
+            if (echo > /dev/tcp/127.0.0.1/4318) >/dev/null 2>&1; then
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "OTel Collector port 4318 not reachable in time" >&2
+          docker compose -p agl-clickhouse -f compose.clickhouse-otel.yml logs otelcol || true
+          exit 1
+
       - name: Prepare artifact directory
         run: mkdir -p "$ARTIFACT_DIR"
 
+      - name: Kafka topic offsets (before workload)
+        if: ${{ matrix.trace_sink == 'kafka' }}
+        run: |
+          set -euo pipefail
+          cd docker
+          # Print offsets for common topic spellings; at least one should exist.
+          {
+            echo "== Kafka offsets BEFORE workload =="
+            # docker compose -p agl-kafka -f compose.kafka-otel.yml exec -T kafka \
+            #   kafka-run-class kafka.tools.GetOffsetShell --broker-list kafka:9092 --topic agl_otlp_spans 2>/dev/null || true
+            docker compose -p agl-kafka -f compose.kafka-otel.yml exec -T kafka \
+              kafka-run-class kafka.tools.GetOffsetShell --broker-list kafka:9092 --topic agl-otlp-spans 2>/dev/null || true
+          } | tee "$GITHUB_WORKSPACE/$ARTIFACT_DIR/kafka-offsets-before.txt"
+
+
+      - name: ClickHouse row count (before workload)
+        if: ${{ matrix.trace_sink == 'clickhouse' }}
+        run: |
+          set -euo pipefail
+          cd docker
+          {
+            echo "== ClickHouse rows BEFORE workload =="
+            docker compose -p agl-clickhouse -f compose.clickhouse-otel.yml exec -T clickhouse               clickhouse-client -q "SELECT count() FROM otel.otel_traces" 2>/dev/null || true
+          } | tee "$GITHUB_WORKSPACE/$ARTIFACT_DIR/clickhouse-rows-before.txt"
+
       - name: Record workload start
         run: echo "BENCHMARK_START=$(date -u +%FT%TZ)" >> "$GITHUB_ENV"
 
@@ -260,6 +471,30 @@ jobs:
         if: ${{ always() }}
         run: echo "BENCHMARK_END=$(date -u +%FT%TZ)" >> "$GITHUB_ENV"
 
+      - name: Kafka topic offsets (after workload)
+        if: ${{ always() && matrix.trace_sink == 'kafka' }}
+        run: |
+          set -euo pipefail
+          cd docker
+          {
+            echo "== Kafka offsets AFTER workload =="
+            # docker compose -p agl-kafka -f compose.kafka-otel.yml exec -T kafka \
+            #   kafka-run-class kafka.tools.GetOffsetShell --broker-list kafka:9092 --topic agl_otlp_spans 2>/dev/null || true
+            docker compose -p agl-kafka -f compose.kafka-otel.yml exec -T kafka \
+              kafka-run-class kafka.tools.GetOffsetShell --broker-list kafka:9092 --topic agl-otlp-spans 2>/dev/null || true
+          } | tee "$GITHUB_WORKSPACE/$ARTIFACT_DIR/kafka-offsets-after.txt"
+
+
+      - name: ClickHouse row count (after workload)
+        if: ${{ always() && matrix.trace_sink == 'clickhouse' }}
+        run: |
+          set -euo pipefail
+          cd docker
+          {
+            echo "== ClickHouse rows AFTER workload =="
+            docker compose -p agl-clickhouse -f compose.clickhouse-otel.yml exec -T clickhouse               clickhouse-client -q "SELECT count() FROM otel.otel_traces" 2>/dev/null || true
+          } | tee "$GITHUB_WORKSPACE/$ARTIFACT_DIR/clickhouse-rows-after.txt"
+
       - name: Show micro benchmark summary
         if: ${{ always() && matrix.workload.kind == 'micro' }}
         run: |
@@ -303,6 +538,51 @@ jobs:
             docker compose -f "$COMPOSE_FILE" logs "$service" > "../$ARTIFACT_DIR/docker-${service}-${WORKLOAD_ID}-${BACKEND_ID}.log" || true
           done
 
+
+      - name: Collect Kafka + OTel Collector logs
+        if: ${{ always() && matrix.trace_sink == 'kafka' }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$ARTIFACT_DIR"
+          cd docker
+          if [ -f compose.kafka-otel.yml ]; then
+            for service in zookeeper kafka otelcol; do
+              docker compose -p agl-kafka -f compose.kafka-otel.yml logs "$service" \
+                > "../$ARTIFACT_DIR/docker-kafka-${service}-${WORKLOAD_ID}-${BACKEND_ID}.log" || true
+            done
+          fi
+
+
+      - name: Collect ClickHouse + OTel Collector logs
+        if: ${{ always() && matrix.trace_sink == 'clickhouse' }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$ARTIFACT_DIR"
+          cd docker
+          if [ -f compose.clickhouse-otel.yml ]; then
+            for service in clickhouse otelcol; do
+              docker compose -p agl-clickhouse -f compose.clickhouse-otel.yml logs "$service"                 > "../$ARTIFACT_DIR/docker-clickhouse-${service}-${WORKLOAD_ID}-${BACKEND_ID}.log" || true
+            done
+          fi
+
+      - name: Stop Kafka + OTel Collector
+        if: ${{ always() && matrix.trace_sink == 'kafka' }}
+        run: |
+          set -euo pipefail
+          cd docker
+          if [ -f compose.kafka-otel.yml ]; then
+            docker compose -p agl-kafka -f compose.kafka-otel.yml down -v || true
+          fi
+
+      - name: Stop ClickHouse + OTel Collector
+        if: ${{ always() && matrix.trace_sink == 'clickhouse' }}
+        run: |
+          set -euo pipefail
+          cd docker
+          if [ -f compose.clickhouse-otel.yml ]; then
+            docker compose -p agl-clickhouse -f compose.clickhouse-otel.yml down -v || true
+          fi
+
       - name: Stop ${{ matrix.backend.id }} Prometheus stack
         if: ${{ always() }}
         run: |
@@ -327,115 +607,115 @@ jobs:
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: error
 
-  collection-benchmarks:
-    name: collection (${{ matrix.backend.id }}, ${{ matrix.workload.id }})
-    runs-on: ${{ matrix.backend.runner }}
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      matrix:
-        backend:
-          - id: memory
-            needs_mongo: false
-            runner: ubuntu-latest
-          - id: mongo
-            needs_mongo: true
-            runner: ubuntu-latest
-        workload:
-          - id: high-insert
-            total_tasks: 50000
-            concurrency: 2048
-            type: insert
-          - id: medium-insert
-            total_tasks: 50000
-            concurrency: 128
-            type: insert
-          - id: low-insert
-            total_tasks: 50000
-            concurrency: 4
-            type: insert
-          - id: high-dequeue
-            total_tasks: 50000
-            concurrency: 2048
-            type: dequeue
-          - id: medium-dequeue
-            total_tasks: 50000
-            concurrency: 128
-            type: dequeue
-          - id: low-dequeue
-            total_tasks: 50000
-            concurrency: 4
-            type: dequeue
-    env:
-      ARTIFACT_DIR: ${{ format('artifacts/{0}-{1}', matrix.backend.id, matrix.workload.id) }}
-      SUMMARY_FILE: ${{ format('artifacts/{0}-{1}/summary-{0}-{1}.jsonl', matrix.backend.id, matrix.workload.id) }}
-      ARTIFACT_NAME: ${{ format('collections-{0}-{1}', matrix.backend.id, matrix.workload.id) }}
-      MONGO_URI: mongodb://localhost:27017/?replicaSet=rs0
-    steps:
-      - uses: actions/checkout@v4
+  # collection-benchmarks:
+  #   name: collection (${{ matrix.backend.id }}, ${{ matrix.workload.id }})
+  #   runs-on: ${{ matrix.backend.runner }}
+  #   timeout-minutes: 15
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       backend:
+  #         - id: memory
+  #           needs_mongo: false
+  #           runner: ubuntu-latest
+  #         - id: mongo
+  #           needs_mongo: true
+  #           runner: ubuntu-latest
+  #       workload:
+  #         - id: high-insert
+  #           total_tasks: 50000
+  #           concurrency: 2048
+  #           type: insert
+  #         - id: medium-insert
+  #           total_tasks: 50000
+  #           concurrency: 128
+  #           type: insert
+  #         - id: low-insert
+  #           total_tasks: 50000
+  #           concurrency: 4
+  #           type: insert
+  #         - id: high-dequeue
+  #           total_tasks: 50000
+  #           concurrency: 2048
+  #           type: dequeue
+  #         - id: medium-dequeue
+  #           total_tasks: 50000
+  #           concurrency: 128
+  #           type: dequeue
+  #         - id: low-dequeue
+  #           total_tasks: 50000
+  #           concurrency: 4
+  #           type: dequeue
+  #   env:
+  #     ARTIFACT_DIR: ${{ format('artifacts/{0}-{1}', matrix.backend.id, matrix.workload.id) }}
+  #     SUMMARY_FILE: ${{ format('artifacts/{0}-{1}/summary-{0}-{1}.jsonl', matrix.backend.id, matrix.workload.id) }}
+  #     ARTIFACT_NAME: ${{ format('collections-{0}-{1}', matrix.backend.id, matrix.workload.id) }}
+  #     MONGO_URI: mongodb://localhost:27017/?replicaSet=rs0
+  #   steps:
+  #     - uses: actions/checkout@v4
 
-      - uses: astral-sh/setup-uv@v7
-        with:
-          enable-cache: true
-          python-version: '3.12'
+  #     - uses: astral-sh/setup-uv@v7
+  #       with:
+  #         enable-cache: true
+  #         python-version: '3.12'
 
-      - name: Sync dependencies
-        run: uv sync --frozen --extra mongo --group core-stable --group dev
+  #     - name: Sync dependencies
+  #       run: uv sync --frozen --extra mongo --group core-stable --group dev
 
-      - name: Launch MongoDB
-        if: ${{ matrix.backend.needs_mongo }}
-        run: |
-          set -euo pipefail
-          cd docker
-          docker compose -f compose.mongo.yml down -v || true
-          docker compose -f compose.mongo.yml up -d --quiet-pull
-          for attempt in {1..60}; do
-            if docker compose -f compose.mongo.yml exec -T mongo mongosh --quiet --eval 'db.runCommand({ping:1})' >/dev/null 2>&1; then
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "MongoDB did not become ready in time" >&2
-          docker compose -f compose.mongo.yml logs mongo
-          exit 1
+  #     - name: Launch MongoDB
+  #       if: ${{ matrix.backend.needs_mongo }}
+  #       run: |
+  #         set -euo pipefail
+  #         cd docker
+  #         docker compose -f compose.mongo.yml down -v || true
+  #         docker compose -f compose.mongo.yml up -d --quiet-pull
+  #         for attempt in {1..60}; do
+  #           if docker compose -f compose.mongo.yml exec -T mongo mongosh --quiet --eval 'db.runCommand({ping:1})' >/dev/null 2>&1; then
+  #             exit 0
+  #           fi
+  #           sleep 2
+  #         done
+  #         echo "MongoDB did not become ready in time" >&2
+  #         docker compose -f compose.mongo.yml logs mongo
+  #         exit 1
 
-      - name: Run collection benchmark
-        run: |
-          set -euo pipefail
-          mkdir -p "$ARTIFACT_DIR"
-          echo "Running collection benchmark (backend=${{ matrix.backend.id }}, workload=${{ matrix.workload.id }})"
-          uv run --locked --no-sync python -m tests.benchmark.collection_benchmark \
-            "${{ matrix.workload.type }}" \
-            --backend "${{ matrix.backend.id }}" \
-            --total-tasks "${{ matrix.workload.total_tasks }}" \
-            --concurrency "${{ matrix.workload.concurrency }}" \
-            --task-prefix "${{ matrix.backend.id }}-${{ matrix.workload.id }}" \
-            --summary-file "$SUMMARY_FILE" \
-            --mongo-uri "$MONGO_URI" \
-            --mongo-database agentlightning_collection_bench
+  #     - name: Run collection benchmark
+  #       run: |
+  #         set -euo pipefail
+  #         mkdir -p "$ARTIFACT_DIR"
+  #         echo "Running collection benchmark (backend=${{ matrix.backend.id }}, workload=${{ matrix.workload.id }})"
+  #         uv run --locked --no-sync python -m tests.benchmark.collection_benchmark \
+  #           "${{ matrix.workload.type }}" \
+  #           --backend "${{ matrix.backend.id }}" \
+  #           --total-tasks "${{ matrix.workload.total_tasks }}" \
+  #           --concurrency "${{ matrix.workload.concurrency }}" \
+  #           --task-prefix "${{ matrix.backend.id }}-${{ matrix.workload.id }}" \
+  #           --summary-file "$SUMMARY_FILE" \
+  #           --mongo-uri "$MONGO_URI" \
+  #           --mongo-database agentlightning_collection_bench
 
-      - name: Show collection benchmark summary
-        if: ${{ always() }}
-        run: |
-          set -euo pipefail
-          if [ -f "$SUMMARY_FILE" ]; then
-            echo "Collection benchmark summary (${{ matrix.backend.id }}):"
-            cat "$SUMMARY_FILE"
-          else
-            echo "Summary file not found: $SUMMARY_FILE"
-          fi
+  #     - name: Show collection benchmark summary
+  #       if: ${{ always() }}
+  #       run: |
+  #         set -euo pipefail
+  #         if [ -f "$SUMMARY_FILE" ]; then
+  #           echo "Collection benchmark summary (${{ matrix.backend.id }}):"
+  #           cat "$SUMMARY_FILE"
+  #         else
+  #           echo "Summary file not found: $SUMMARY_FILE"
+  #         fi
 
-      - name: Stop MongoDB
-        if: ${{ always() && matrix.backend.needs_mongo }}
-        run: |
-          set -euo pipefail
-          cd docker
-          docker compose -f compose.mongo.yml down -v || true
+  #     - name: Stop MongoDB
+  #       if: ${{ always() && matrix.backend.needs_mongo }}
+  #       run: |
+  #         set -euo pipefail
+  #         cd docker
+  #         docker compose -f compose.mongo.yml down -v || true
 
-      - name: Upload collection artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ env.ARTIFACT_NAME }}
-          path: ${{ env.ARTIFACT_DIR }}
-          if-no-files-found: error
+  #     - name: Upload collection artifacts
+  #       if: ${{ always() }}
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: ${{ env.ARTIFACT_NAME }}
+  #         path: ${{ env.ARTIFACT_DIR }}
+  #         if-no-files-found: error

README.md

@@ -37,7 +37,7 @@ pip install agentlightning
 For the latest nightly build (cutting-edge features), you can install from Test PyPI:
 
 ```bash
-pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ agentlightning
+pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ --pre agentlightning
 ```
 
 Please refer to our [installation guide](https://microsoft.github.io/agent-lightning/stable/tutorials/installation/) for more details.
@@ -46,6 +46,7 @@ To start using Agent-lightning, check out our [documentation](https://microsoft.
 
 ## ⚡ Articles
 
+- 12/17/2025 [Adopting the Trajectory Level Aggregation for Faster Training](https://agent-lightning.github.io/posts/trajectory_level_aggregation/) Agent-lightning blog.
 - 11/4/2025 [Tuning ANY AI agent with Tinker ✕ Agent-lightning](https://medium.com/@yugez/tuning-any-ai-agent-with-tinker-agent-lightning-part-1-1d8c9a397f0e) Medium. See also [Part 2](https://medium.com/@yugez/tuning-any-ai-agent-with-tinker-agent-lightning-part-2-332c5437f0dc).
 - 10/22/2025 [No More Retokenization Drift: Returning Token IDs via the OpenAI Compatible API Matters in Agent RL](https://blog.vllm.ai/2025/10/22/agent-lightning.html) vLLM blog. See also [Zhihu writeup](https://zhuanlan.zhihu.com/p/1965067274642785725).
 - 8/11/2025 [Training AI Agents to Write and Self-correct SQL with Reinforcement Learning](https://medium.com/@yugez/training-ai-agents-to-write-and-self-correct-sql-with-reinforcement-learning-571ed31281ad) Medium.

agentlightning/__init__.py

@@ -1,6 +1,6 @@
 # Copyright (c) Microsoft. All rights reserved.
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 
 from .adapter import *
 from .algorithm import *

agentlightning/algorithm/apo/apo.py

@@ -112,6 +112,8 @@ class APO(Algorithm, Generic[T_task]):
         beam_rounds: int = 3,
         rollout_batch_timeout: float = 3600.0,
         run_initial_validation: bool = True,
+        gradient_prompt_files: Optional[List[Path]] = None,
+        apply_edit_prompt_files: Optional[List[Path]] = None,
         # Internal flags for debugging
         _poml_trace: bool = False,
     ):
@@ -132,6 +134,8 @@ class APO(Algorithm, Generic[T_task]):
             rollout_batch_timeout: Maximum time in seconds to wait for rollout batch completion.
             run_initial_validation: If True, runs validation on the seed prompt before starting
                 optimization to establish a baseline score. Defaults to True.
+            gradient_prompt_files: Prompt templates used to compute textual gradients (critiques).
+            apply_edit_prompt_files: Prompt templates used to apply edits based on critiques.
         """
         self.async_openai_client = async_openai_client
         self.gradient_model = gradient_model
@@ -144,6 +148,8 @@ class APO(Algorithm, Generic[T_task]):
         self.beam_rounds = beam_rounds
         self.rollout_batch_timeout = rollout_batch_timeout
         self.run_initial_validation = run_initial_validation
+        self.gradient_prompt_files = gradient_prompt_files or GRADIENT_PROMPT_FILES
+        self.apply_edit_prompt_files = apply_edit_prompt_files or APPLY_EDIT_PROMPT_FILES
 
         self._history_best_prompt: Optional[PromptTemplate] = None
         self._history_best_score: float = float("-inf")
@@ -270,7 +276,7 @@ class APO(Algorithm, Generic[T_task]):
         Returns:
             A textual critique generated by the LLM, or None if generation fails.
         """
-        tg_template = random.choice(GRADIENT_PROMPT_FILES)
+        tg_template = random.choice(self.gradient_prompt_files)
 
         if len(rollout_results) < self.gradient_batch_size:
             self._log(
@@ -352,7 +358,7 @@ class APO(Algorithm, Generic[T_task]):
             return current_prompt.prompt_template.template
 
         # 2) Apply edit
-        ae_template = random.choice(APPLY_EDIT_PROMPT_FILES)
+        ae_template = random.choice(self.apply_edit_prompt_files)
         self._log(
             logging.INFO,
             f"Edit will be generated by {self.apply_edit_model} with template: {ae_template.name}",

agentlightning/algorithm/verl/interface.py

@@ -41,15 +41,19 @@ class VERL(Algorithm):
         ```python
         config["agentlightning"]["trace_aggregator"] = {
             "level": "trajectory",
-            "trajectory_max_prompt_length": ...,
-            "trajectory_max_response_length": ...,
+            "trajectory_max_prompt_length": 4096,
+            "trajectory_max_response_length": 34384,
         }
         ```
 
         Keep conversations structured (message lists rather than manual string
-        concatenation) so prefix matching can stitch traces, and toggle `debug=True` plus
-        `unmatch_log_dir` when you need to inspect retokenization or chat-template
-        mismatches. See [this blog post](https://agent-lightning.github.io/posts/trajectory_level_aggregation/)
+        concatenation) so prefix matching can stitch traces. `trajectory_max_prompt_length`
+        should be set to the maximum length of the prompt for the first turn, and
+        `trajectory_max_response_length` should be set to the maximum cumulative
+        length of agent responses in the full trajectory.
+        Toggle `debug=True` plus `mismatch_log_dir` when you need to inspect
+        retokenization or chat-template mismatches. See
+        [this blog post](https://agent-lightning.github.io/posts/trajectory_level_aggregation/)
         for more details.
 
     Examples:

agentlightning/tracer/otel.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import os
 import threading
 import warnings
 from contextlib import asynccontextmanager, contextmanager
@@ -122,7 +123,7 @@ class OtelTracer(Tracer):
 
     @with_active_tracer_context
     @asynccontextmanager
-    async def trace_context(
+    async def trace_context(  # kh: runner.step_impl에서 옴
         self,
         name: Optional[str] = None,
         *,
@@ -160,6 +161,7 @@ class OtelTracer(Tracer):
             if store.capabilities.get("otlp_traces", False) is True:
                 logger.debug(f"Tracing to LightningStore rollout_id={rollout_id}, attempt_id={attempt_id}")
                 self._enable_native_otlp_exporter(store, rollout_id, attempt_id)
+                # kh: otlp traces 지원하는 store인 경우, otlp exporter 사용
             else:
                 self._disable_native_otlp_exporter()
             ctx = self._lightning_span_processor.with_context(store=store, rollout_id=rollout_id, attempt_id=attempt_id)
@@ -262,16 +264,33 @@ class OtelTracer(Tracer):
         )
         instrumented = False
         candidates: List[str] = []
+        # kh: processor 별로 조건 check 하는 것임!!
         for processor in active_span_processor._span_processors:  # pyright: ignore[reportPrivateUsage]
             if isinstance(processor, LightningSpanProcessor):
                 # We don't need the LightningSpanProcessor any more.
                 logger.debug("LightningSpanProcessor already present in TracerProvider, disabling it.")
                 processor.disable_store_submission = True
+
+                # kh added
+                logger.debug(f"disable_store_submission set to True for rollout={rollout_id} attempt={attempt_id}")
             elif isinstance(processor, (SimpleSpanProcessor, BatchSpanProcessor)):
                 # Instead, we rely on the OTLPSpanExporter to send spans to the store.
                 if isinstance(processor.span_exporter, LightningStoreOTLPExporter):
-                    processor.span_exporter.enable_store_otlp(store.otlp_traces_endpoint(), rollout_id, attempt_id)
-                    logger.debug(f"Set LightningStoreOTLPExporter endpoint to {store.otlp_traces_endpoint()}")
+                    # option 1
+                    # kh 기존: store로 보낸다는 설정;
+                    # processor.span_exporter.enable_store_otlp(store.otlp_traces_endpoint(), rollout_id, attempt_id)
+                    # logger.debug(f"Set LightningStoreOTLPExporter endpoint to {store.otlp_traces_endpoint()}")
+
+                    # option 2 (hardcoded for collector)
+                    # collector 실험: otel collector로 설정?
+                    # processor.span_exporter.enable_store_otlp("http://localhost:4318/v1/traces", rollout_id, attempt_id)
+                    # logger.info(f"Set LightningStoreOTLPExporter endpoint to http://localhost:4318/v1/traces")
+
+                    # option 3 (switchable via env var)
+                    endpoint = os.getenv("AGL_OTLP_ENDPOINT") or store.otlp_traces_endpoint()
+                    processor.span_exporter.enable_store_otlp(endpoint, rollout_id, attempt_id)
+                    logger.debug(f"[SET] enabling span export to {endpoint} rollout={rollout_id} attempt={attempt_id}")
+
                     instrumented = True
                 else:
                     candidates.append(
@@ -288,6 +307,9 @@ class OtelTracer(Tracer):
             )
 
     def _disable_native_otlp_exporter(self):
+        # kh added
+        # logger.info("disable_store_submission reset to False")
+
         tracer_provider = self._get_tracer_provider()
         active_span_processor = tracer_provider._active_span_processor  # pyright: ignore[reportPrivateUsage]
         tracer_provider._resource = tracer_provider._resource.merge(  # pyright: ignore[reportPrivateUsage]
@@ -480,12 +502,21 @@ class LightningSpanProcessor(SpanProcessor):
         if not span.context or not span.context.trace_flags.sampled:
             return
 
+        # kh: _disable_store_submission 조건에 안 맞아서 안 들어갈 가능성 큼
         if not self._disable_store_submission and self._store and self._rollout_id and self._attempt_id:
             try:
                 # Submit add_otel_span to the event loop and wait for it to complete
                 with suppress_instrumentation():
                     self._ensure_loop()
+
+                    # kh added
+                    # logger.info(
+                    #     "[SET] Exporting spans to STORE; rollout=%s attempt=%s",
+                    #     span.name, self._rollout_id, self._attempt_id
+                    # )
+
                     uploaded_span = self._await_in_loop(
+                        # kh: store에 span 저장 시도 (이건 otlp collector로 보내는 게 아님, store에 직접!)
                         self._store.add_otel_span(self._rollout_id, self._attempt_id, span),
                         timeout=STORE_WRITE_TIMEOUT_SECONDS,
                     )

agentlightning/verl/config.yaml

@@ -13,7 +13,7 @@ agentlightning:
     trajectory_max_prompt_length: 2048  # supported in trajectory level aggregation, suggest to set as maximum length for the prompt in first turn
     trajectory_max_response_length: 8192  # supported in trajectory level aggregation, suggest to set as maximum length for the cumulative agent responses in the full trajectory, i.e., n_turns * (max_response_length + max_prompt_length)
     debug: False  # supported in trajectory level aggregation, enable to diagnose trace merging failures
-    unmatch_log_dir: ./unmatch_cases  # supported in trajectory level aggregation with debug=True, directory to store logs of unmatched cases
+    mismatch_log_dir: ./mismatch_cases  # supported in trajectory level aggregation with debug=True, directory to store logs of mismatch cases
 
 data:
   filter_overlong_prompts: false

agentlightning/verl/daemon.py

@@ -945,7 +945,7 @@ class AgentModeDaemon:
                             global_steps,
                             rollout_id,
                             turn_index,
-                            self.trace_aggregator.get("unmatch_log_dir", None),
+                            self.trace_aggregator.get("mismatch_log_dir", None),
                         )
 
                     if is_prefix:

dashboard/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "agent-lightning-dashboard",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "agent-lightning-dashboard",
-      "version": "0.3.0",
+      "version": "0.3.1",
       "dependencies": {
         "@mantine/core": "8.3.5",
         "@mantine/hooks": "8.3.5",

dashboard/package.json

@@ -1,7 +1,7 @@
 {
   "name": "agent-lightning-dashboard",
   "type": "module",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "scripts": {
     "dev": "vite",
     "build": "tsc && vite build",

docs/changelog.md

@@ -1,5 +1,97 @@
 # Changelog
 
+## Agent-lightning v0.3.0 (12/24/2025)
+
+Agent-lightning v0.3.0 is a major release that introduces several new features and bug fixes. The release is a collaborative effort between Agent-lightning core teams and the community. Thanks to all the contributors who made this release possible.
+
+### Highlights
+
+* **Tinker integration**: Support Tinker as an alternative backend for Reinforcement Learning (#226 #245 #264 #269 #327). See [example code](https://github.com/microsoft/agent-lightning/tree/v0.3.0/examples/tinker), [blog 1](https://medium.com/@yugez/tuning-any-ai-agent-with-tinker-agent-lightning-part-1-1d8c9a397f0e) and [blog 2](https://medium.com/@yugez/tuning-any-ai-agent-with-tinker-agent-lightning-part-2-332c5437f0dc).
+* **Azure OpenAI integration**: Support Azure OpenAI as a backend for LLM inference and supervised fine-tuning (#256 #327). [Example code](https://github.com/microsoft/agent-lightning/tree/v0.3.0/examples/azure).
+* **MongoDB-based Lightning Store** is added as an alternative backend for Lightning Store (#323). [Documentation](https://microsoft.github.io/agent-lightning/0.3.0/tutorials/parallelize/#parallelizing-lightningstore).
+* **Contrib package**: Add contrib package for community projects. Search-R1 is integrated as a contrib recipe. More coming. (#239 #396 #410 #412 #417).
+* **RESTful API**: Stabilize and document RESTful API for Lightning Store (#241 #275). [Documentation](https://microsoft.github.io/agent-lightning/0.3.0/reference/restful/).
+* **OTel Semantic Conventions** that are specifically designed for Agent-optimization areas (#340). [Documentation](https://microsoft.github.io/agent-lightning/0.3.0/reference/semconv/).
+* *[Preview]* **Agent-lightning Dashboard** is now available (#288 #289 #291 #296 #371 #375). It's the official web application for inspecting and debugging Agent-lightning experiments. See details [here](https://microsoft.github.io/agent-lightning/0.3.0/tutorials/debug/).
+* *[Preview]* **Multi-modality example** featuring VERL and a LangGraph agent on ChartQA dataset (#379). [Example code](https://github.com/microsoft/agent-lightning/tree/v0.3.0/examples/chartqa).
+* *[Preview]* Integrate **Claude Code** as a LitAgent and support training on SWE-Bench (#332 #346 #348). [Example code](https://github.com/microsoft/agent-lightning/tree/v0.3.0/examples/claude_code).
+* *[Preview]* **Weave tracer** as a substitute for AgentOps tracer (#277 #411 #420 #423). [Documentation](https://microsoft.github.io/agent-lightning/0.3.0/tutorials/traces/#weave-tracer-experimental).
+* *[Preview]* **Trajectory Level Aggregation** for more efficient training with VERL. See [blog](https://agent-lightning.github.io/posts/trajectory_level_aggregation/) and [documentation](https://microsoft.github.io/agent-lightning/0.3.0/algorithm-zoo/verl/).
+
+### Store Benchmark
+
+In this release, the Lightning Store core was redesigned for significantly greater efficiency and scalability (#315 #318 #328 #342 #344 #356 #380 #388 #418 #421). The benchmark results below demonstrate the impact: with large numbers of concurrent runners, v0.3.0 delivers up to a 15x increase in throughput compared to v0.2.2.
+
+| Throughput (\#rollout/sec) | v0.2.2 | v0.3.0 (in-memory) | v0.3.0 (Mongo) |
+| :---- | :---- | :---- | :---- |
+| Minimal (batch, #runner=32, #turns=6) | 8.73 | 9.06 | 8.71 |
+| Medium (batch, #runners=100, #turns=10) | 12.03 | 23.26 | 32.79 |
+| Mid-high (batch, #runners=300, #turns=6) | 10.61 | 24.42 | 40.24 |
+| Large (batch, #runners=1000, #turns=3) | 3.36 | 14.60 | 50.05 |
+| Long queue (queue, #runners=256, #turns=4) | 7.42 | 30.86 | 57.01 |
+| Heavy trace (queue, #runners=512, #turns=20) | 5.93 | 13.28 | 29.41 |
+
+*Notes:*
+
+1. Benchmarks were run on a single Standard_D32as_v4 Azure VM (Large and heavy trace tests used Standard_D64ads_v5), executed via GitHub Actions.
+2. Two algorithm patterns are evaluated: the batch pattern submits a group of rollouts and waits for all to finish before starting the next group, while the queue pattern maintains a set number of in-flight rollouts, submitting new ones as soon as capacity frees up. Configuration details are available [here](https://github.com/microsoft/agent-lightning/blob/v0.3.0/.github/workflows/benchmark.yml).
+3. The number of turns is directly proportional to the number of spans each rollout generates.
+
+### Maintenance and Bug fixes
+
+#### Core (Store, Interfaces, etc.)
+
+* Add Trainer port option for client-server strategies (#198)
+* Fix store port conflict handling (#227)
+* Unified PythonServerLauncher (#286 #292 #303)
+* Make health timeout configurable (#305)
+* Refactor logging (#306)
+* Support OTLP in LightningStore (#313)
+* Centralized metrics helper (#368)
+* Fix redundant cancel tracebacks on Ctrl+C (#370)
+
+#### Proxy, Adapters and Algorithms
+
+* Fix training metrics before and after processing in VERL (#145)
+* Forward streaming requests for Anthropic and OpenAI APIs (as non-streaming requests) (#299)
+* Check traces with reward for VERL (#317)
+* Patch LiteLLM root span (#341)
+* Handle ref_in_actor flag for LoRA compatibility (#386)
+* Support `with_llm_proxy` and `with_store` in algorithms (#398)
+* Support image URL export in TracerTraceToTriplets (#400)
+* Fix match_rewards assign_to elements in TraceTree (#403)
+* Support customizing trainer and daemon in VERL (#407)
+
+#### Runners, Tracers and Agents
+
+* Refactor tracer initialization (#321)
+* Fix OpenAI Agents 0.6 compatibility (#322)
+* `emit_operation`, `emit_annotation`, tags and links (#359)
+* Sunset HTTP tracer (#402)
+
+#### Examples
+
+* Fix typos in train-first-agent.md (#263)
+* Fix room_selector example which always runs the first task (#270)
+* Fix typo in SQL agent example (#285)
+* Add the README and script files for training SQL agent on NPU (#272)
+* Examples Catalog and Refine Contribution Guide (#331)
+* Upgrade LangChain to 1.x (#364)
+* Update RAG example to Agent-lightning v0.2.x (#349)
+
+#### Miscellaneous
+
+* DeepWiki Badge (#263)
+* Add AGENTS.md (#374)
+
+### New Contributors
+
+Warm welcome to our first-time contributors: @cptnm3, @TerryChan, @genji970, @zxgx, @xiaochulaoban, @lspinheiro, @Kwanghoon-Choi, @Vasuk12, @totoluo, @jinghuan-Chen 🎉
+
+**Full Changelog**: https://github.com/microsoft/agent-lightning/compare/v0.2.0...v0.3.0
+
+---
+
 ## Agent-lightning v0.2.2 (11/12/2025)
 
 Agent-lightning v0.2.2 is a stabilization release for v0.2.1. It introduces several bug fixes.

docs/reference/restful.md

@@ -1,10 +1,8 @@
 # RESTful API References
 
-!!! warning
+!!! note
 
-    The following contents are still under construction.
-
-## Store RESTful API
+    Shown in the following is the RESTful API for Lightning Store.
 
 <div id="swagger-ui"></div>
 <link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist/swagger-ui.css" />

docs/tutorials/installation.md

@@ -34,7 +34,7 @@ This installs or upgrades Agent-Lightning to the newest stable version.
 Agent-Lightning also publishes **nightly builds**, which contain the latest experimental features and improvements from the main branch. These are available via **Test PyPI**.
 
 ```bash
-pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ agentlightning
+pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ --pre agentlightning
 ```
 
 !!! warning

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentlightning"
-version = "0.3.0"
+version = "0.3.1"
 description = "Agent-lightning is the absolute trainer to light up AI agents."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -328,19 +328,24 @@ include = [
   "agentlightning/**/*.yaml",
   "agentlightning/**/*.yml",
   "agentlightning/**/*.poml",
-  "agentlightning/**/*.html",
-  "agentlightning/**/*.js",
-  "agentlightning/**/*.css",
-  "agentlightning/**/*.svg",
+]
+artifacts = [
+  "agentlightning/dashboard/**",
 ]
 
 [tool.hatch.build.targets.sdist]
 exclude = [
-  "examples/**",
-  "tests/**",
-  "docs/**",
-  "scripts/**",
-  "dashboard/**",
+  "/examples/**",
+  "/tests/**",
+  "/docs/**",
+  "/scripts/**",
+  "/dashboard/**",
+  "/docker/**",
+  "/contrib/**",
+  "/.github/**",
+]
+artifacts = [
+  "agentlightning/dashboard/**",
 ]
 
 [tool.pytest.ini_options]

tests/benchmark/benchmark_store.py

@@ -93,6 +93,8 @@ def make_agent(max_rounds: int, sleep_seconds: float) -> agl.LitAgent[str]:
         rounds = random.randint(1, max_rounds)
         selected_round = random.randint(0, rounds - 1)
 
+        # kh: 각 span이 with block에서 종료되면, otlp SDK가 SpanProcessor.on_end(span) 호출,
+        # kh: 내부적으로 span_exporter.export, 미리 지정한 endpoint(collector)로 span 전송 시도.
         for i in range(rounds):
             with tracer.start_as_current_span(f"agent{i}") as span:
                 # Nested Span
@@ -211,8 +213,11 @@ class AlgorithmBatch(agl.Algorithm):
                         continue
                     if rollout.status != "succeeded":
                         raise RuntimeError(f"Rollout {rollout_id} finished with status {rollout.status}")
-                    spans = await store.query_spans(rollout_id=rollout_id, attempt_id="latest")
-                    check_spans(spans, pending[rollout_id])
+
+                    # kh: Skipping reading from store
+                    # spans = await store.query_spans(rollout_id=rollout_id, attempt_id="latest")
+                    # check_spans(spans, pending[rollout_id])
+
                     completed_ids.add(rollout_id)
                     complete_ids_updated = True
 
@@ -407,6 +412,7 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
                 "managed_store": False,
             },
         )
+        # kh: make_agent에서 정의한 agent가 들어감. 그리고 이 agent는 매 rollout 마다 round, span, attribute 생성하고 sleep도 호출.
         trainer.fit(make_agent(max_rounds=args.max_rounds, sleep_seconds=args.sleep_seconds))
     finally:
         timeout_guard.cancel()

uv.lock

@@ -155,7 +155,7 @@ wheels = [
 
 [[package]]
 name = "agentlightning"
-version = "0.3.0"
+version = "0.3.1"
 source = { editable = "." }
 dependencies = [
     { name = "agentops", version = "0.4.18", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'group-14-agentlightning-core-legacy') or (sys_platform == 'linux' and extra == 'group-14-agentlightning-torch-gpu-legacy') or (sys_platform == 'linux' and extra == 'group-14-agentlightning-torch-legacy') or (extra == 'group-14-agentlightning-core-legacy' and extra == 'group-14-agentlightning-core-stable') or (extra == 'group-14-agentlightning-core-legacy' and extra == 'group-14-agentlightning-tinker') or (extra == 'group-14-agentlightning-core-legacy' and extra == 'group-14-agentlightning-torch-gpu-stable') or (extra == 'group-14-agentlightning-core-legacy' and extra == 'group-14-agentlightning-torch-stable') or (extra == 'group-14-agentlightning-core-legacy' and extra == 'group-14-agentlightning-trl') or (extra == 'group-14-agentlightning-core-legacy' and extra == 'group-14-agentlightning-vllm-0-10-2') or (extra == 'group-14-agentlightning-core-legacy' and extra == 'group-14-agentlightning-vllm-0-11-0') or (extra == 'group-14-agentlightning-core-stable' and extra == 'group-14-agentlightning-torch-gpu-legacy') or (extra == 'group-14-agentlightning-core-stable' and extra == 'group-14-agentlightning-torch-legacy') or (extra == 'group-14-agentlightning-langchain' and extra == 'group-14-agentlightning-torch-gpu-legacy') or (extra == 'group-14-agentlightning-langchain' and extra == 'group-14-agentlightning-torch-legacy') or (extra == 'group-14-agentlightning-tinker' and extra == 'group-14-agentlightning-torch-gpu-legacy') or (extra == 'group-14-agentlightning-tinker' and extra == 'group-14-agentlightning-torch-legacy') or (extra == 'group-14-agentlightning-torch-gpu-legacy' and extra == 'group-14-agentlightning-torch-gpu-stable') or (extra == 'group-14-agentlightning-torch-gpu-legacy' and extra == 'group-14-agentlightning-torch-stable') or (extra == 'group-14-agentlightning-torch-gpu-legacy' and extra == 'group-14-agentlightning-trl') or (extra == 'group-14-agentlightning-torch-gpu-legacy' and extra == 'group-14-agentlightning-vllm-0-10-2') or (extra == 'group-14-agentlightning-torch-gpu-legacy' and extra == 'group-14-agentlightning-vllm-0-11-0') or (extra == 'group-14-agentlightning-torch-gpu-stable' and extra == 'group-14-agentlightning-torch-legacy') or (extra == 'group-14-agentlightning-torch-legacy' and extra == 'group-14-agentlightning-torch-stable') or (extra == 'group-14-agentlightning-torch-legacy' and extra == 'group-14-agentlightning-trl') or (extra == 'group-14-agentlightning-torch-legacy' and extra == 'group-14-agentlightning-vllm-0-10-2') or (extra == 'group-14-agentlightning-torch-legacy' and extra == 'group-14-agentlightning-vllm-0-11-0') or (extra == 'group-14-agentlightning-torch-cpu' and extra == 'group-14-agentlightning-torch-cu128') or (extra == 'group-14-agentlightning-torch-cpu' and extra == 'group-14-agentlightning-torch-gpu-legacy') or (extra == 'group-14-agentlightning-torch-cpu' and extra == 'group-14-agentlightning-torch-gpu-stable') or (extra == 'group-14-agentlightning-vllm-0-10-2' and extra == 'group-14-agentlightning-vllm-0-11-0')" },