Ling-Sen Peng
11ee26616c
test_llm_judge_validates_compiled_workflow fails: the judge returns pass=false, reporting structural items from the kitchen-sink agent spec as missing from the compiled workflowDef. Listed as a plain reason string rather than run:false — the test makes a single judge call and fails fast, so there is no CI time to reclaim, and leaving it running means a fix surfaces as XPASS. E2E_MIN_PASSED drops 135 -> 134 in the same commit, as the known-failures README requires: an added entry moves a test out of the PASSED column, and without the matching decrement the lane fails on the passed-count floor for an unrelated-looking reason. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
767 lines
33 KiB
YAML
767 lines
33 KiB
YAML
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
paths-ignore:
|
|
- "conductor-clients/**"
|
|
pull_request:
|
|
paths-ignore:
|
|
- "conductor-clients/**"
|
|
workflow_dispatch:
|
|
inputs:
|
|
e2e_profiles:
|
|
description: 'E2E profiles to run in addition to redis-es8 (which always runs): "all", "none", or a comma separated subset of postgres, mysql, redis-os3'
|
|
required: false
|
|
default: all
|
|
type: string
|
|
# GitHub Actions cron expressions are UTC.
|
|
schedule:
|
|
- cron: "0 2 * * *"
|
|
|
|
# A new push to a PR supersedes that PR's in-flight run - cancel it instead of
|
|
# burning runners on a stale sha (same pattern as orkes-conductor's workflows).
|
|
# Groups are separated by event type so nightly/dispatch/push runs never
|
|
# cross-cancel each other or PR runs.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
jobs:
|
|
detect-changes:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
persistence: ${{ steps.filter.outputs.persistence }}
|
|
ui: ${{ steps.filter.outputs.ui }}
|
|
agent: ${{ steps.filter.outputs.agent }}
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
fetch-depth: 0
|
|
- uses: dorny/paths-filter@v3
|
|
id: filter
|
|
with:
|
|
filters: |
|
|
persistence:
|
|
- 'cassandra-persistence/**'
|
|
- 'common-persistence/**'
|
|
- 'es6-persistence/**'
|
|
- 'es7-persistence/**'
|
|
- 'es8-persistence/**'
|
|
- 'mysql-persistence/**'
|
|
- 'os-persistence/**'
|
|
- 'os-persistence-v2/**'
|
|
- 'os-persistence-v3/**'
|
|
- 'postgres-persistence/**'
|
|
- 'redis-persistence/**'
|
|
- 'scheduler/cassandra-persistence/**'
|
|
- 'scheduler/mysql-persistence/**'
|
|
- 'scheduler/postgres-persistence/**'
|
|
- 'scheduler/redis-persistence/**'
|
|
- 'scheduler/sqlite-persistence/**'
|
|
- 'sqlite-persistence/**'
|
|
ui:
|
|
- 'ui/**'
|
|
- 'ui-next/**'
|
|
agent:
|
|
- 'ai/**'
|
|
# Directory is 'agentspan/'; ':conductor-agentspan' is only its Gradle
|
|
# project name, so a 'conductor-agentspan/**' pattern matches nothing.
|
|
- 'agentspan/**'
|
|
- 'server/**'
|
|
- 'core/**'
|
|
- 'sqlite-persistence/**'
|
|
- '.github/agent-e2e/**'
|
|
- '.github/workflows/ci.yml'
|
|
|
|
build:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
fetch-depth: 0
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /opt/ghc
|
|
- name: Gradle wrapper validation
|
|
uses: gradle/wrapper-validation-action@v3
|
|
- name: Set up Zulu JDK 21
|
|
uses: actions/setup-java@v5
|
|
with:
|
|
distribution: "zulu"
|
|
java-version: "21"
|
|
- name: Cache SonarCloud packages
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: ~/.sonar/cache
|
|
key: ${{ runner.os }}-sonar
|
|
restore-keys: ${{ runner.os }}-sonar
|
|
- name: Cache Gradle packages
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: ${{ runner.os }}-gradle-
|
|
- name: Build with Gradle
|
|
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
|
run: |
|
|
./gradlew build -x :conductor-test-harness:test -x test
|
|
- name: Build and Publish snapshot
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
run: |
|
|
echo "Running build for commit ${{ github.sha }}"
|
|
./gradlew build -x :conductor-test-harness:test -x test
|
|
- name: Generate aggregated coverage report
|
|
if: always()
|
|
run: ./gradlew jacocoAggregatedReport -x test || true
|
|
- name: Publish Test Report
|
|
uses: mikepenz/action-junit-report@v6
|
|
if: always()
|
|
with:
|
|
report_paths: "**/build/test-results/test/TEST-*.xml"
|
|
- name: Upload build artifacts
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: build-artifacts
|
|
path: "**/build/reports"
|
|
- name: Upload coverage report
|
|
uses: actions/upload-artifact@v7
|
|
if: always()
|
|
with:
|
|
name: coverage-report
|
|
path: build/reports/jacoco/aggregated
|
|
unit-test:
|
|
needs: detect-changes
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
checks: write
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
fetch-depth: 0
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /opt/ghc
|
|
- name: Set up Zulu JDK 21
|
|
uses: actions/setup-java@v5
|
|
with:
|
|
distribution: "zulu"
|
|
java-version: "21"
|
|
- name: Cache Gradle packages
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: ${{ runner.os }}-gradle-
|
|
- name: Force Docker API Version
|
|
run: echo 'api.version=1.44' > ~/.docker-java.properties
|
|
- name: Run unit tests
|
|
run: |
|
|
# Manual dispatch and the nightly build are the full validation paths.
|
|
if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "schedule" ]; then
|
|
./gradlew test -x :conductor-test-harness:test
|
|
exit 0
|
|
fi
|
|
# Push and PR CI run persistence tests only when one of the persistence modules changed.
|
|
# Direct Gradle invocations (for example, :conductor-postgres-persistence:test) remain
|
|
# unaffected; this gate applies only to the aggregate CI command.
|
|
if [ "${{ needs.detect-changes.outputs.persistence }}" == "true" ]; then
|
|
./gradlew test -x :conductor-test-harness:test
|
|
else
|
|
./gradlew test \
|
|
-x :conductor-test-harness:test \
|
|
-x :conductor-cassandra-persistence:test \
|
|
-x :conductor-common-persistence:test \
|
|
-x :conductor-scheduler-cassandra-persistence:test \
|
|
-x :conductor-es6-persistence:test \
|
|
-x :conductor-es7-persistence:test \
|
|
-x :conductor-es8-persistence:test \
|
|
-x :conductor-mysql-persistence:test \
|
|
-x :conductor-scheduler-mysql-persistence:test \
|
|
-x :conductor-os-persistence:test \
|
|
-x :conductor-os-persistence-v2:test \
|
|
-x :conductor-os-persistence-v3:test \
|
|
-x :conductor-postgres-persistence:test \
|
|
-x :conductor-redis-persistence:test \
|
|
-x :conductor-scheduler-postgres-persistence:test \
|
|
-x :conductor-scheduler-redis-persistence:test \
|
|
-x :conductor-scheduler-sqlite-persistence:test \
|
|
-x :conductor-sqlite-persistence:test
|
|
fi
|
|
- name: Publish Test Report
|
|
uses: mikepenz/action-junit-report@v6
|
|
if: always()
|
|
with:
|
|
check_name: "Unit Test results"
|
|
check_title_template: "{{SUITE_NAME}} | {{TEST_NAME}} | {{CLASS_NAME}}"
|
|
report_paths: "**/build/test-results/test/TEST-*.xml"
|
|
check_retries: true
|
|
detailed_summary: true
|
|
include_passed: false
|
|
flaky_summary: true
|
|
- name: Upload unit-test reports
|
|
uses: actions/upload-artifact@v7
|
|
if: always()
|
|
with:
|
|
name: unit-test-reports
|
|
path: "**/build/reports/tests"
|
|
test-harness:
|
|
runs-on: ubuntu-latest
|
|
# checks: write is required for the test-result check below; without it the
|
|
# default read-only token fails with "Resource not accessible by integration".
|
|
permissions:
|
|
contents: read
|
|
checks: write
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
fetch-depth: 0
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /opt/ghc
|
|
- name: Set up Zulu JDK 21
|
|
uses: actions/setup-java@v5
|
|
with:
|
|
distribution: "zulu"
|
|
java-version: "21"
|
|
- name: Cache Gradle packages
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: ${{ runner.os }}-gradle-
|
|
- name: Force Docker API Version
|
|
run: echo 'api.version=1.44' > ~/.docker-java.properties
|
|
- name: Cache Docker images
|
|
uses: actions/cache@v5
|
|
id: docker-cache
|
|
with:
|
|
path: /tmp/docker-images-test-harness.tar
|
|
key: docker-test-harness-v2
|
|
- name: Load cached Docker images
|
|
if: steps.docker-cache.outputs.cache-hit == 'true'
|
|
run: docker load -i /tmp/docker-images-test-harness.tar || true
|
|
- name: Run test-harness tests
|
|
run: |
|
|
./gradlew :conductor-test-harness:test
|
|
- name: Save Docker images for cache
|
|
if: steps.docker-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
docker image prune -f
|
|
mapfile -t images < <((docker images --format '{{.Repository}}:{{.Tag}}' \
|
|
| grep -v '<none>' \
|
|
| grep -E '(^|/)(elasticsearch|redis|postgres|mysql|mongo|cassandra)(:|/)|mockserver/mockserver|opensearchproject/opensearch|testcontainers/|orkesio/') || true)
|
|
if [ "${#images[@]}" -eq 0 ]; then
|
|
echo "No Testcontainers-related images to cache; writing empty tar for cache action."
|
|
tar -cf /tmp/docker-images-test-harness.tar --files-from /dev/null
|
|
exit 0
|
|
fi
|
|
printf '%s\n' "${images[@]}"
|
|
docker save -o /tmp/docker-images-test-harness.tar "${images[@]}"
|
|
- name: Publish Test Report
|
|
uses: mikepenz/action-junit-report@v6
|
|
if: always()
|
|
with:
|
|
check_name: "Test-Harness results"
|
|
check_title_template: "{{SUITE_NAME}} | {{TEST_NAME}} | {{CLASS_NAME}}"
|
|
report_paths: "test-harness/build/test-results/test/TEST-*.xml"
|
|
check_retries: true
|
|
detailed_summary: true
|
|
include_passed: false
|
|
flaky_summary: true
|
|
- name: Upload test-harness reports
|
|
uses: actions/upload-artifact@v7
|
|
if: always()
|
|
with:
|
|
name: test-harness-reports
|
|
path: "test-harness/build/reports"
|
|
- name: Upload test-harness coverage report
|
|
uses: actions/upload-artifact@v7
|
|
if: always()
|
|
with:
|
|
name: test-harness-coverage-report
|
|
path: "test-harness/build/reports/jacoco"
|
|
agentspan-e2e:
|
|
if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
|
|
runs-on: ubuntu-latest
|
|
# checks: write is required for the test-result check below; without it the
|
|
# default read-only token fails with "Resource not accessible by integration".
|
|
permissions:
|
|
contents: read
|
|
checks: write
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
fetch-depth: 0
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /opt/ghc
|
|
- name: Set up Zulu JDK 21
|
|
uses: actions/setup-java@v5
|
|
with:
|
|
distribution: "zulu"
|
|
java-version: "21"
|
|
- name: Cache Gradle packages
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: ${{ runner.os }}-gradle-
|
|
- name: Force Docker API Version
|
|
run: echo 'api.version=1.44' > ~/.docker-java.properties
|
|
- name: Run deterministic AgentSpan E2E tests
|
|
run: ./gradlew :conductor-test-harness:agentspanDeterministicE2E
|
|
- name: Run live-provider AgentSpan E2E coverage
|
|
if: github.ref == 'refs/heads/main'
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
OPENAI_MODEL: ${{ vars.OPENAI_MODEL }}
|
|
run: ./gradlew :conductor-agentspan:agentspanEndToEndCoverage
|
|
- name: Publish AgentSpan E2E test report
|
|
uses: mikepenz/action-junit-report@v6
|
|
if: always()
|
|
with:
|
|
report_paths: |
|
|
test-harness/build/test-results/agentspanDeterministicE2E/TEST-*.xml
|
|
test-harness/build/test-results/agentspanRealE2E/TEST-*.xml
|
|
check_name: AgentSpan E2E Test Report
|
|
- name: Upload AgentSpan E2E reports
|
|
uses: actions/upload-artifact@v7
|
|
if: always()
|
|
with:
|
|
name: agentspan-e2e-reports
|
|
path: |
|
|
test-harness/build/reports/tests/agentspanDeterministicE2E
|
|
test-harness/build/reports/tests/agentspanRealE2E
|
|
if-no-files-found: warn
|
|
- name: Upload AgentSpan coverage report
|
|
uses: actions/upload-artifact@v7
|
|
if: github.ref == 'refs/heads/main'
|
|
with:
|
|
name: agentspan-e2e-coverage-report
|
|
path: agentspan/build/reports/jacoco/agentspanEndToEndCoverage
|
|
if-no-files-found: error
|
|
generate-e2e-matrix:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
|
steps:
|
|
- id: set-matrix
|
|
shell: bash
|
|
env:
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
E2E_PROFILES: ${{ inputs.e2e_profiles }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
# redis-es8 runs on every trigger, so every push and pull request gets one full
|
|
# end-to-end run. It is listed first in the matrix.
|
|
# redis-es7 and cassandra-es7 are disabled: ES7 coverage is superseded by the
|
|
# redis-es8 flavor and cassandra support is partial. Re-enable by adding them back
|
|
# to the optional list (scripts remain in e2e/).
|
|
always='[{"name":"redis-es8","script":"./e2e/run_tests-es8.sh"}]'
|
|
optional='[{"name":"postgres","script":"./e2e/run_tests-postgres.sh"},
|
|
{"name":"mysql","script":"./e2e/run_tests-mysql.sh"},
|
|
{"name":"redis-os3","script":"./e2e/run_tests-redis-os3.sh"}]'
|
|
|
|
case "$EVENT_NAME" in
|
|
workflow_dispatch) selection="${E2E_PROFILES:-all}" ;;
|
|
schedule) selection="all" ;;
|
|
*) selection="none" ;;
|
|
esac
|
|
|
|
case "$selection" in
|
|
all) extra="$optional" ;;
|
|
none) extra='[]' ;;
|
|
*)
|
|
# Comma separated subset, e.g. "mysql,postgres". redis-es8 is always added.
|
|
unknown=$(jq -rn --argjson optional "$optional" --arg sel "$selection" '
|
|
($sel | split(",") | map(gsub("^\\s+|\\s+$";"")) | map(select(length > 0))) as $want
|
|
| (($optional | map(.name)) + ["redis-es8"]) as $known
|
|
| [$want[] | select(. as $n | ($known | any(. == $n)) | not)] | join(", ")')
|
|
if [ -n "$unknown" ]; then
|
|
echo "::error::Unknown e2e profile(s): $unknown. Valid values: all, none, redis-es8, $(jq -r --argjson o "$optional" -n '$o | map(.name) | join(", ")')"
|
|
exit 1
|
|
fi
|
|
extra=$(jq -c --arg sel "$selection" '
|
|
($sel | split(",") | map(gsub("^\\s+|\\s+$";""))) as $want
|
|
| map(select(.name as $n | $want | any(. == $n)))' <<< "$optional")
|
|
;;
|
|
esac
|
|
|
|
include=$(jq -cn --argjson always "$always" --argjson extra "$extra" '$always + $extra')
|
|
echo "Selected e2e profiles: $(jq -r 'map(.name) | join(", ")' <<< "$include")"
|
|
echo "matrix={\"include\":$include}" >> "$GITHUB_OUTPUT"
|
|
|
|
e2e:
|
|
needs: generate-e2e-matrix
|
|
if: ${{ needs.generate-e2e-matrix.outputs.matrix != '{"include":[]}' }}
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
checks: write
|
|
strategy:
|
|
fail-fast: false
|
|
# All backends in parallel — each matrix job gets its own runner VM, so a full
|
|
# 6-flavor matrix completes in ~15 minutes instead of ~90 sequential.
|
|
matrix: ${{ fromJson(needs.generate-e2e-matrix.outputs.matrix) }}
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
fetch-depth: 0
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /opt/ghc
|
|
- name: Set up Zulu JDK 21
|
|
uses: actions/setup-java@v5
|
|
with:
|
|
distribution: "zulu"
|
|
java-version: "21"
|
|
- name: Cache Gradle packages
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
|
|
restore-keys: ${{ runner.os }}-gradle-
|
|
- name: Run E2E tests (${{ matrix.name }})
|
|
run: ${{ matrix.script }}
|
|
- name: Publish Test Report
|
|
uses: mikepenz/action-junit-report@v6
|
|
if: always()
|
|
with:
|
|
check_name: "E2E results (${{ matrix.name }})"
|
|
check_title_template: "{{SUITE_NAME}} | {{TEST_NAME}} | {{CLASS_NAME}}"
|
|
report_paths: "e2e/build/test-results/test/TEST-*.xml"
|
|
check_retries: true
|
|
detailed_summary: true
|
|
include_passed: false
|
|
flaky_summary: true
|
|
- name: Generate test summary table
|
|
if: always()
|
|
shell: python3 {0}
|
|
run: |
|
|
import os, glob, xml.etree.ElementTree as ET
|
|
|
|
xml_files = glob.glob("e2e/build/test-results/test/TEST-*.xml")
|
|
rows = []
|
|
totals = {"passed": 0, "failed": 0, "skipped": 0}
|
|
|
|
for f in sorted(xml_files):
|
|
try:
|
|
root = ET.parse(f).getroot()
|
|
except ET.ParseError:
|
|
continue
|
|
for tc in root.iter("testcase"):
|
|
name = tc.get("name", "?")
|
|
classname = tc.get("classname", "").split(".")[-1]
|
|
duration = float(tc.get("time", 0))
|
|
if tc.find("skipped") is not None:
|
|
status, totals["skipped"] = "⏭ skip", totals["skipped"] + 1
|
|
elif tc.find("failure") is not None or tc.find("error") is not None:
|
|
node = tc.find("failure") if tc.find("failure") is not None else tc.find("error")
|
|
msg = (node.get("message") or "")[:120]
|
|
status, totals["failed"] = f"❌ `{msg}`", totals["failed"] + 1
|
|
else:
|
|
status, totals["passed"] = "✅", totals["passed"] + 1
|
|
rows.append((classname, name, f"{duration:.1f}s", status))
|
|
|
|
backend = "${{ matrix.name }}"
|
|
lines = [
|
|
f"## E2E results — {backend}",
|
|
f"**✅ {totals['passed']} passed · ❌ {totals['failed']} failed · ⏭ {totals['skipped']} skipped**",
|
|
"",
|
|
"| Class | Test | Duration | Result |",
|
|
"|-------|------|----------|--------|",
|
|
]
|
|
for cls, name, dur, status in rows:
|
|
lines.append(f"| {cls} | {name} | {dur} | {status} |")
|
|
|
|
summary = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/null")
|
|
with open(summary, "a") as fh:
|
|
fh.write("\n".join(lines) + "\n")
|
|
- name: Upload E2E reports
|
|
uses: actions/upload-artifact@v7
|
|
if: always()
|
|
with:
|
|
name: e2e-reports-${{ matrix.name }}
|
|
path: "e2e/build/reports"
|
|
|
|
build-ui:
|
|
# ui-next is the default UI: it is what docker/server/Dockerfile bundles
|
|
# into the conductor:server image, so it is what CI builds and tests.
|
|
needs: detect-changes
|
|
if: >-
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.event_name == 'schedule' ||
|
|
needs.detect-changes.outputs.ui == 'true'
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
working-directory: ui-next
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: Set up pnpm
|
|
uses: pnpm/action-setup@v4
|
|
with:
|
|
package_json_file: ui-next/package.json
|
|
|
|
- name: Set up Node
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: "22"
|
|
cache: "pnpm"
|
|
cache-dependency-path: ui-next/pnpm-lock.yaml
|
|
|
|
- name: Install Dependencies
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: Typecheck
|
|
run: pnpm typecheck
|
|
|
|
- name: Unit tests (vitest)
|
|
run: pnpm test
|
|
|
|
- name: Build UI
|
|
run: NODE_OPTIONS=--max-old-space-size=4096 pnpm build
|
|
|
|
# Boots the conductor server built from THIS commit in SQLite mode (the default — no external
|
|
# DB) and runs the released python SDK agent e2e suite against it, so a server change can't
|
|
# silently break the SDK before a release. The suite + bundle come from conductor-oss/python-sdk
|
|
# (conductor-ai-e2e-python-<version>); the server auto-configures the openai provider from
|
|
# OPENAI_API_KEY (conductor.ai.openai.api-key), so no manual integration setup is needed.
|
|
# Known failures are xfail-ed via .github/agent-e2e/known-failures-python.json so the lane can
|
|
# gate while a gap is fixed; see that file for the current list and why each is listed.
|
|
# Runs on push/dispatch/schedule, and on same-repo PRs that touch agent-relevant paths.
|
|
# Fork PRs are skipped on purpose: this is a public repo and GitHub withholds secrets from
|
|
# fork-originated workflows, so OPENAI_API_KEY would be empty, the server's openai provider
|
|
# would fail to initialise, and the LLM suites would fail for a reason the contributor
|
|
# cannot fix. The fork test is nested inside the pull_request branch — hoisting it into a
|
|
# top-level AND would also disable push/dispatch/schedule, where github.event.pull_request
|
|
# is null and the comparison is therefore false.
|
|
python-sdk-e2e:
|
|
name: Python SDK E2E (SQLite)
|
|
needs: detect-changes
|
|
if: >-
|
|
github.event_name != 'pull_request' ||
|
|
(needs.detect-changes.outputs.agent == 'true' &&
|
|
github.event.pull_request.head.repo.full_name == github.repository)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 40
|
|
permissions:
|
|
contents: read
|
|
checks: write # publish the junit test-report check
|
|
env:
|
|
# Pinned python-sdk version. The suite is not downloaded as a released bundle — the SDK
|
|
# repo is checked out at the tag derived from this value and its e2e/ is run in place
|
|
# (see run-suite.sh), so the SDK under test is built from that source rather than
|
|
# resolved from PyPI. python-sdk release tags carry no `v` prefix.
|
|
#
|
|
# Because nothing has to be published for a ref to work, this can be any tag, branch or
|
|
# SHA — set the `CONDUCTOR_PYTHON_E2E_BUNDLE_VERSION` repo/org variable to point the lane
|
|
# at unreleased SDK work without a commit here. The `_BUNDLE_` in the name is vestigial;
|
|
# it matches the repository variable already configured in GitHub settings.
|
|
CONDUCTOR_PYTHON_E2E_BUNDLE_VERSION: ${{ vars.CONDUCTOR_PYTHON_E2E_BUNDLE_VERSION || '2.0.0-rc4' }}
|
|
# The names the suite's conftest reads. Verify against the pinned ref before renaming:
|
|
# a name the suite does not read falls through to conftest's defaults silently, so the
|
|
# lane would run green against the wrong server and model.
|
|
CONDUCTOR_SERVER_URL: http://localhost:8080/api
|
|
CONDUCTOR_AGENT_LLM_MODEL: openai/gpt-4o-mini
|
|
MCP_TESTKIT_URL: http://localhost:3001
|
|
# Suite21 probes GET {this}/scheduler/schedules and skips the whole suite unless it
|
|
# answers 200; its own default is port 8089. conductor-oss serves the scheduler itself
|
|
# (conductor.scheduler.enabled=true by default, SchedulerResource at /api/scheduler),
|
|
# so point it at the same server or the scheduling tests silently do not run.
|
|
SCHEDULER_CONDUCTOR_URL: http://localhost:8080/api
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
E2E_KNOWN_FAILURES: ${{ github.workspace }}/.github/agent-e2e/known-failures-python.json
|
|
# KNOWN RISK, accepted: run-suite.sh pip-installs at run time from the SDK's own
|
|
# package metadata, which bounds only its direct deps — pytest, langgraph and the
|
|
# provider SDKs all float. An upstream release can therefore redden this lane with no
|
|
# change to this repo, and it will present as a server regression rather than a
|
|
# dependency bump. If this lane fails for no apparent reason, suspect that first and
|
|
# check the pip install output in the log before hunting a server cause. (Pinning the
|
|
# transitive set via PIP_CONSTRAINT works, but the lockfile has to be regenerated on
|
|
# every SDK bump, which was judged not worth the upkeep.)
|
|
# Minimum number of tests that must actually PASS. A pytest skip is not a failure, so a
|
|
# server change that breaks a test's setup silently converts pass -> skip and the lane
|
|
# stays green while covering less. This floor is the backstop: if fewer than this many
|
|
# tests pass, fail regardless of whether anything reported a failure. RAISE it when real
|
|
# coverage is added; LOWER it only with a written reason — a silent lowering is exactly
|
|
# the regression this guards against. Moving the pin can change the test set, so
|
|
# re-measure against the new ref rather than assuming this number still fits.
|
|
E2E_MIN_PASSED: "134"
|
|
steps:
|
|
- uses: actions/checkout@v7
|
|
|
|
- name: Set up Zulu JDK 21
|
|
uses: actions/setup-java@v5
|
|
with:
|
|
distribution: zulu
|
|
java-version: "21"
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Build server boot jar
|
|
run: ./gradlew :conductor-server:bootJar -x test --no-daemon
|
|
|
|
- name: Start mcp-testkit (best effort)
|
|
run: |
|
|
# Pinned: mcp-testkit 1.0.3 declares `mcp[cli]>=1.0.0` with no upper bound, and mcp
|
|
# 2.0.0 dropped mcp.server.fastmcp, which the testkit imports at start-up — so an
|
|
# unpinned install yields a testkit that exits immediately and every MCP suite skips.
|
|
python -m pip install --quiet 'mcp-testkit==1.0.3' 'mcp<2' || { echo "::warning::mcp-testkit install failed — MCP suites will skip"; exit 0; }
|
|
nohup mcp-testkit --transport http --port 3001 > mcp-testkit.log 2>&1 &
|
|
# Readiness = "the port answers HTTP at all", NOT 2xx. mcp-testkit serves 404 on /,
|
|
# so `curl -sf` (which treats 404 as failure) never succeeded: the loop always ran
|
|
# all 15 iterations and the message below printed unconditionally, whether or not
|
|
# anything was listening. Drop -f and check we got a status line instead.
|
|
for i in $(seq 1 15); do
|
|
# Fallback outside the substitution: on connection failure curl already prints
|
|
# "000" AND exits non-zero, so `|| echo 000` inside would concatenate to "000000"
|
|
# and read as a live response. The `||` also keeps `bash -e` from aborting here.
|
|
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 2 http://localhost:3001/)" || code="000"
|
|
if [ "$code" != "000" ]; then echo "mcp-testkit responding (HTTP $code) after ~${i}s"; break; fi
|
|
if [ "$i" -eq 15 ]; then
|
|
echo "::warning::mcp-testkit did not respond within 15s — MCP suites will skip"
|
|
tail -20 mcp-testkit.log || true
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
- name: Start conductor server (SQLite)
|
|
run: |
|
|
JAR="$(ls server/build/libs/conductor-server-*-boot.jar | head -1)"
|
|
echo "booting $JAR"
|
|
nohup java -jar "$JAR" --server.port=8080 > conductor-server.log 2>&1 &
|
|
for i in $(seq 1 60); do
|
|
if curl -sf http://localhost:8080/health >/dev/null 2>&1; then
|
|
echo "server healthy after ~$((i * 3))s"; break
|
|
fi
|
|
if [ "$i" -eq 60 ]; then echo "::error::server failed to become healthy within 180s"; tail -100 conductor-server.log; exit 1; fi
|
|
sleep 3
|
|
done
|
|
|
|
- name: Resolve the pinned SDK ref
|
|
id: sdk
|
|
run: |
|
|
V="${CONDUCTOR_PYTHON_E2E_BUNDLE_VERSION}"
|
|
# python-sdk tags bare versions — no `v` prefix.
|
|
echo "ref=${V}" >> "$GITHUB_OUTPUT"
|
|
echo "e2e suite: conductor-oss/python-sdk @ ${V}"
|
|
|
|
# The ref can be any tag, branch or SHA — override the version variable above to point
|
|
# this lane at unreleased SDK work. python-sdk is public, so the default GITHUB_TOKEN
|
|
# is enough.
|
|
- name: Check out the SDK e2e suite at the pinned ref
|
|
uses: actions/checkout@v7
|
|
with:
|
|
repository: conductor-oss/python-sdk
|
|
ref: ${{ steps.sdk.outputs.ref }}
|
|
path: .github/agent-e2e/src-python
|
|
|
|
# run-suite.sh forwards trailing args to pytest, so `-p known_failures_plugin` loads the
|
|
# orkes-style xfail plugin (found via PYTHONPATH) without touching the upstream suite.
|
|
# Gating: a genuine failure fails the job; entries in known-failures-python.json are
|
|
# xfail-ed green. `-rs` prints the reason for every skip in the log — without it a test
|
|
# that quietly stops running is invisible in the output.
|
|
- name: Run python e2e (SQLite; known failures xfail-ed)
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}/.github/agent-e2e
|
|
run: |
|
|
bash .github/agent-e2e/run-suite.sh \
|
|
.github/agent-e2e/src-python -p known_failures_plugin -rs
|
|
|
|
# Skips are free — CI only reddens on failure — so a broken test setup degrades this
|
|
# lane silently. Enforce a floor on the PASSED count so lost coverage is loud.
|
|
- name: Enforce passed-count floor
|
|
working-directory: .github/agent-e2e/src-python
|
|
run: |
|
|
python - <<'PY'
|
|
import os, sys, xml.etree.ElementTree as ET
|
|
|
|
floor = int(os.environ["E2E_MIN_PASSED"])
|
|
path = "results/junit-e2e.xml"
|
|
if not os.path.exists(path):
|
|
sys.exit(f"::error::{path} is missing — the suite produced no results at all")
|
|
|
|
root = ET.parse(path).getroot()
|
|
suites = [root] if root.tag == "testsuite" else root.findall(".//testsuite")
|
|
tests = errors = failures = skipped = 0
|
|
for s in suites:
|
|
tests += int(s.get("tests", 0))
|
|
errors += int(s.get("errors", 0))
|
|
failures += int(s.get("failures", 0))
|
|
skipped += int(s.get("skipped", 0)) # junit lumps xfail in with skipped
|
|
passed = tests - errors - failures - skipped
|
|
|
|
print(f"passed={passed} skipped(+xfail)={skipped} failed={failures} "
|
|
f"errors={errors} total={tests} floor={floor}")
|
|
if passed < floor:
|
|
sys.exit(
|
|
f"::error::only {passed} tests passed, floor is {floor}. Tests that used to "
|
|
f"pass are now skipping (a skip is not a failure, so nothing else catches "
|
|
f"this). Find out why before touching E2E_MIN_PASSED."
|
|
)
|
|
PY
|
|
|
|
- name: Publish test report
|
|
if: always()
|
|
continue-on-error: true # gating is the e2e run step's exit code; a checks-API hiccup must not fail the job
|
|
uses: mikepenz/action-junit-report@v4
|
|
with:
|
|
report_paths: ".github/agent-e2e/src-python/results/junit-e2e.xml"
|
|
check_name: "Python SDK E2E (SQLite)"
|
|
fail_on_failure: false
|
|
require_tests: false
|
|
|
|
- name: Upload results + server log
|
|
if: always()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: python-sdk-e2e-results
|
|
path: |
|
|
.github/agent-e2e/src-python/results/**
|
|
conductor-server.log
|
|
mcp-testkit.log
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|