Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b57c9f175b | |||
| a503a2a8a6 |
@@ -1,19 +1,15 @@
|
||||
{
|
||||
"name": "Python 3",
|
||||
"image": "mcr.microsoft.com/devcontainers/python:3.14-bookworm",
|
||||
"image": "mcr.microsoft.com/devcontainers/python:3.13-bullseye",
|
||||
"features": {
|
||||
"ghcr.io/va-h/devcontainers-features/uv:1": {},
|
||||
"ghcr.io/devcontainers/features/docker-in-docker:3": {},
|
||||
"ghcr.io/devcontainers/features/azure-cli:1.2.9": {},
|
||||
"ghcr.io/devcontainers/features/copilot-cli:1": {}
|
||||
"ghcr.io/devcontainers/features/azure-cli:1.2.8": {}
|
||||
},
|
||||
"postCreateCommand": "bash ./devsetup.sh",
|
||||
"workspaceFolder": "/workspaces/agent-framework/python/",
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
"GitHub.copilot",
|
||||
"GitHub.vscode-github-actions",
|
||||
"ms-python.python",
|
||||
"ms-windows-ai-studio.windows-ai-studio",
|
||||
"littlefoxteam.vscode-python-test-adapter"
|
||||
|
||||
@@ -2,14 +2,13 @@ dirs:
|
||||
- .
|
||||
excludedFiles:
|
||||
- ./python/CHANGELOG.md
|
||||
- "**/SKILL.md"
|
||||
ignorePatterns:
|
||||
- pattern: "/github/"
|
||||
- pattern: "./actions"
|
||||
- pattern: "./blob"
|
||||
- pattern: "./issues"
|
||||
- pattern: "./discussions"
|
||||
- pattern: "./pull"
|
||||
- pattern: "./pulls"
|
||||
- pattern: "https:\/\/platform.openai.com"
|
||||
- pattern: "http:\/\/localhost"
|
||||
- pattern: "http:\/\/127.0.0.1"
|
||||
@@ -21,13 +20,9 @@ ignorePatterns:
|
||||
- pattern: "https://your-resource.openai.azure.com/"
|
||||
- pattern: "http://host.docker.internal"
|
||||
- pattern: "https://openai.github.io/openai-agents-js/openai/agents/classes/"
|
||||
# dotnet.microsoft.com bot-blocks CI link checkers with intermittent 403s on any
|
||||
# path (including localized variants like /en-us/download/...), so ignore the
|
||||
# whole domain rather than just /download.
|
||||
- pattern: "https:\/\/dotnet.microsoft.com"
|
||||
- pattern: "https://github.com/Rel1cx/eslint-react"
|
||||
- pattern: "https:\/\/dotnet.microsoft.com\/download"
|
||||
# excludedDirs:
|
||||
# Folders which include links to localhost, since it's not ignored with regular expressions
|
||||
# Folders which include links to localhost, since it's not ignored with regular expressions
|
||||
baseUrl: https://github.com/microsoft/agent-framework/
|
||||
aliveStatusCodes:
|
||||
- 200
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Code ownership assignments
|
||||
# https://docs.github.com/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
|
||||
|
||||
python/packages/azurefunctions/ @microsoft/agentframework-durabletask-developers
|
||||
python/packages/durabletask/ @microsoft/agentframework-durabletask-developers
|
||||
python/samples/getting_started/azure_functions/ @microsoft/agentframework-durabletask-developers
|
||||
python/samples/getting_started/durabletask/ @microsoft/agentframework-durabletask-developers
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
name: .NET Bug Report
|
||||
description: Report a bug in the Agent Framework .NET SDK
|
||||
title: ".NET: [Bug]: "
|
||||
labels: [".NET"]
|
||||
labels: ["bug", ".NET"]
|
||||
type: bug
|
||||
body:
|
||||
- type: textarea
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
name: Python Bug Report
|
||||
description: Report a bug in the Agent Framework Python SDK
|
||||
title: "Python: [Bug]: "
|
||||
labels: ["Python"]
|
||||
labels: ["bug", "Python"]
|
||||
type: bug
|
||||
body:
|
||||
- type: textarea
|
||||
@@ -47,7 +47,7 @@ body:
|
||||
attributes:
|
||||
label: Package Versions
|
||||
description: List the agent-framework-* packages and versions you are using
|
||||
placeholder: "e.g., agent-framework-core: 1.0.0, agent-framework-foundry: 1.0.0"
|
||||
placeholder: "e.g., agent-framework-core: 1.0.0, agent-framework-azure-ai: 1.0.0"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
name: Azure Functions Integration Test Setup
|
||||
description: Prepare local emulators and tools for Azure Functions integration tests
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Start Durable Task Scheduler Emulator
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$(docker ps -aq -f name=dts-emulator)" ]; then
|
||||
echo "Stopping and removing existing Durable Task Scheduler Emulator"
|
||||
docker rm -f dts-emulator
|
||||
fi
|
||||
echo "Starting Durable Task Scheduler Emulator"
|
||||
docker run -d --name dts-emulator -p 8080:8080 -p 8082:8082 -e DTS_USE_DYNAMIC_TASK_HUBS=true mcr.microsoft.com/dts/dts-emulator:latest
|
||||
echo "Waiting for Durable Task Scheduler Emulator to be ready"
|
||||
timeout 30 bash -c 'until curl --silent http://localhost:8080/healthz; do sleep 1; done'
|
||||
echo "Durable Task Scheduler Emulator is ready"
|
||||
- name: Start Azurite (Azure Storage emulator)
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$(docker ps -aq -f name=azurite)" ]; then
|
||||
echo "Stopping and removing existing Azurite (Azure Storage emulator)"
|
||||
docker rm -f azurite
|
||||
fi
|
||||
echo "Starting Azurite (Azure Storage emulator)"
|
||||
docker run -d --name azurite -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite
|
||||
echo "Waiting for Azurite (Azure Storage emulator) to be ready"
|
||||
timeout 30 bash -c 'until curl --silent http://localhost:10000/devstoreaccount1; do sleep 1; done'
|
||||
echo "Azurite (Azure Storage emulator) is ready"
|
||||
- name: Start Redis
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "$(docker ps -aq -f name=redis)" ]; then
|
||||
echo "Stopping and removing existing Redis"
|
||||
docker rm -f redis
|
||||
fi
|
||||
echo "Starting Redis"
|
||||
docker run -d --name redis -p 6379:6379 redis:latest
|
||||
echo "Waiting for Redis to be ready"
|
||||
timeout 30 bash -c 'until docker exec redis redis-cli ping | grep -q PONG; do sleep 1; done'
|
||||
echo "Redis is ready"
|
||||
- name: Install Azure Functions Core Tools
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Installing Azure Functions Core Tools"
|
||||
npm install -g azure-functions-core-tools@4 --unsafe-perm true
|
||||
func --version
|
||||
@@ -1,64 +0,0 @@
|
||||
name: Free runner disk space
|
||||
description: |
|
||||
Reclaims disk space on GitHub-hosted Ubuntu runners by removing
|
||||
pre-installed toolchains we do not use (Android SDK, GHC/Haskell,
|
||||
CodeQL bundle), Docker images, and swap. Also relocates the
|
||||
NuGet package cache to /mnt (which has ~75 GB free vs ~14 GB
|
||||
on /). No-op on non-Linux runners.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Free disk space (Linux only)
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Disk usage before cleanup"
|
||||
df -h /
|
||||
echo "::endgroup::"
|
||||
|
||||
# Remove pre-installed toolchains we never use on this repo's
|
||||
# dotnet/python jobs. These reclaim ~25-30 GB on ubuntu-latest.
|
||||
sudo rm -rf \
|
||||
/usr/local/lib/android \
|
||||
/usr/share/dotnet/sdk/NuGetFallbackFolder \
|
||||
/opt/ghc \
|
||||
/usr/local/.ghcup \
|
||||
/opt/hostedtoolcache/CodeQL \
|
||||
/opt/hostedtoolcache/PyPy \
|
||||
/opt/hostedtoolcache/Ruby \
|
||||
/opt/hostedtoolcache/go \
|
||||
/usr/local/share/boost \
|
||||
/usr/local/share/powershell \
|
||||
/usr/local/share/chromium \
|
||||
/usr/local/share/vcpkg \
|
||||
/usr/local/lib/heroku \
|
||||
"${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache}/PyPy" \
|
||||
"${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache}/Ruby" \
|
||||
"${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache}/go" || true
|
||||
|
||||
# Drop docker images shipped on the runner; jobs that need
|
||||
# docker pull what they need fresh.
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
sudo docker image prune --all --force >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
# Disable swap to free its backing file.
|
||||
sudo swapoff -a || true
|
||||
sudo rm -f /mnt/swapfile /swapfile || true
|
||||
|
||||
echo "::group::Disk usage after cleanup"
|
||||
df -h /
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Relocate NuGet package cache to /mnt (Linux only)
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo mkdir -p /mnt/nuget
|
||||
sudo chown -R "$USER":"$USER" /mnt/nuget
|
||||
echo "NUGET_PACKAGES=/mnt/nuget" >> "$GITHUB_ENV"
|
||||
echo "Relocated NuGet package cache to /mnt/nuget"
|
||||
df -h /mnt || true
|
||||
@@ -1,112 +0,0 @@
|
||||
name: Get GitHub automation token
|
||||
description: Creates a GitHub App installation token with a temporary PAT fallback
|
||||
|
||||
inputs:
|
||||
mode:
|
||||
description: Authentication mode (app, app-with-fallback, or pat)
|
||||
required: false
|
||||
default: app-with-fallback
|
||||
azure-client-id:
|
||||
description: Client ID of the Azure workload identity
|
||||
required: false
|
||||
azure-tenant-id:
|
||||
description: Azure tenant ID
|
||||
required: false
|
||||
azure-subscription-id:
|
||||
description: Azure subscription containing the Key Vault
|
||||
required: false
|
||||
key-vault-name:
|
||||
description: Azure Key Vault name
|
||||
required: false
|
||||
key-name:
|
||||
description: Key Vault key used to sign the GitHub App JWT
|
||||
required: false
|
||||
github-app-client-id:
|
||||
description: GitHub App client ID
|
||||
required: false
|
||||
github-app-installation-id:
|
||||
description: GitHub App installation ID
|
||||
required: false
|
||||
repository:
|
||||
description: Repository to include in the installation token
|
||||
required: false
|
||||
fallback-token:
|
||||
description: PAT used temporarily when app authentication is unavailable
|
||||
required: false
|
||||
|
||||
outputs:
|
||||
token:
|
||||
description: GitHub App installation token or fallback PAT
|
||||
value: ${{ steps.select-token.outputs.token }}
|
||||
source:
|
||||
description: Selected authentication source
|
||||
value: ${{ steps.select-token.outputs.source }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Validate authentication mode
|
||||
shell: bash
|
||||
env:
|
||||
AUTH_MODE: ${{ inputs.mode || 'app-with-fallback' }}
|
||||
run: |
|
||||
if [[ "$AUTH_MODE" != "app" && "$AUTH_MODE" != "app-with-fallback" && "$AUTH_MODE" != "pat" ]]; then
|
||||
echo "::error::Unsupported GitHub authentication mode."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Sign in to Azure
|
||||
id: azure-login
|
||||
if: ${{ (inputs.mode || 'app-with-fallback') != 'pat' }}
|
||||
continue-on-error: true
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
with:
|
||||
client-id: ${{ inputs.azure-client-id }}
|
||||
tenant-id: ${{ inputs.azure-tenant-id }}
|
||||
subscription-id: ${{ inputs.azure-subscription-id }}
|
||||
|
||||
- name: Create GitHub App installation token
|
||||
id: app-token
|
||||
if: ${{ (inputs.mode || 'app-with-fallback') != 'pat' && steps.azure-login.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
env:
|
||||
AZURE_SUBSCRIPTION_ID: ${{ inputs.azure-subscription-id }}
|
||||
KEY_VAULT_NAME: ${{ inputs.key-vault-name }}
|
||||
KEY_NAME: ${{ inputs.key-name }}
|
||||
GITHUB_APP_CLIENT_ID: ${{ inputs.github-app-client-id }}
|
||||
GITHUB_APP_INSTALLATION_ID: ${{ inputs.github-app-installation-id }}
|
||||
TARGET_REPOSITORY: ${{ inputs.repository }}
|
||||
run: |
|
||||
token="$(node "$GITHUB_ACTION_PATH/create-token.js")"
|
||||
echo "::add-mask::$token"
|
||||
echo "token=$token" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Select authentication token
|
||||
id: select-token
|
||||
shell: bash
|
||||
env:
|
||||
AUTH_MODE: ${{ inputs.mode || 'app-with-fallback' }}
|
||||
APP_TOKEN: ${{ steps.app-token.outputs.token }}
|
||||
FALLBACK_TOKEN: ${{ inputs.fallback-token }}
|
||||
run: |
|
||||
if [[ "$AUTH_MODE" != "pat" && -n "$APP_TOKEN" ]]; then
|
||||
token="$APP_TOKEN"
|
||||
source="app"
|
||||
echo "::notice::GitHub authentication source: app"
|
||||
elif [[ "$AUTH_MODE" == "app-with-fallback" && -n "$FALLBACK_TOKEN" ]]; then
|
||||
token="$FALLBACK_TOKEN"
|
||||
source="pat-fallback"
|
||||
echo "::warning::GitHub authentication source: PAT fallback"
|
||||
elif [[ "$AUTH_MODE" == "pat" && -n "$FALLBACK_TOKEN" ]]; then
|
||||
token="$FALLBACK_TOKEN"
|
||||
source="pat-forced"
|
||||
echo "::warning::GitHub authentication source: PAT (forced rollout mode)"
|
||||
else
|
||||
echo "::error::GitHub App authentication is unavailable and no fallback PAT was provided."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::add-mask::$token"
|
||||
echo "token=$token" >> "$GITHUB_OUTPUT"
|
||||
echo "source=$source" >> "$GITHUB_OUTPUT"
|
||||
@@ -1,133 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
const crypto = require('node:crypto');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
|
||||
function base64Url(value) {
|
||||
return Buffer.from(value).toString('base64url');
|
||||
}
|
||||
|
||||
function base64ToBase64Url(value) {
|
||||
return Buffer.from(value, 'base64').toString('base64url');
|
||||
}
|
||||
|
||||
function createJwtSigningInput(clientId, nowSeconds) {
|
||||
const header = base64Url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }));
|
||||
const payload = base64Url(JSON.stringify({
|
||||
iat: nowSeconds - 60,
|
||||
exp: nowSeconds + 540,
|
||||
iss: clientId,
|
||||
}));
|
||||
return `${header}.${payload}`;
|
||||
}
|
||||
|
||||
function signJwt(signingInput, config, execute = execFileSync) {
|
||||
const digest = crypto.createHash('sha256').update(signingInput).digest('base64');
|
||||
const signature = execute(
|
||||
'az',
|
||||
[
|
||||
'keyvault', 'key', 'sign',
|
||||
'--subscription', config.azureSubscriptionId,
|
||||
'--vault-name', config.keyVaultName,
|
||||
'--name', config.keyName,
|
||||
'--algorithm', 'RS256',
|
||||
'--digest', digest,
|
||||
'--query', 'signature',
|
||||
'--output', 'tsv',
|
||||
'--only-show-errors',
|
||||
],
|
||||
{ encoding: 'utf8' },
|
||||
).trim();
|
||||
|
||||
if (!signature) {
|
||||
throw new Error('Key Vault returned an empty signature.');
|
||||
}
|
||||
|
||||
return `${signingInput}.${base64ToBase64Url(signature)}`;
|
||||
}
|
||||
|
||||
async function createInstallationToken(config, dependencies = {}) {
|
||||
const execute = dependencies.execute ?? execFileSync;
|
||||
const request = dependencies.fetch ?? fetch;
|
||||
const nowSeconds = dependencies.nowSeconds ?? Math.floor(Date.now() / 1000);
|
||||
const repositoryParts = config.targetRepository.split('/');
|
||||
|
||||
if (repositoryParts.length !== 2 || repositoryParts.some((part) => part.length === 0)) {
|
||||
throw new Error('TARGET_REPOSITORY must use the owner/repository format.');
|
||||
}
|
||||
|
||||
const [, repository] = repositoryParts;
|
||||
const signingInput = createJwtSigningInput(config.githubAppClientId, nowSeconds);
|
||||
const jwt = signJwt(signingInput, config, execute);
|
||||
|
||||
const response = await request(
|
||||
`https://api.github.com/app/installations/${config.githubAppInstallationId}/access_tokens`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${jwt}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
repositories: [repository],
|
||||
permissions: {
|
||||
contents: 'read',
|
||||
issues: 'write',
|
||||
members: 'read',
|
||||
pull_requests: 'write',
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`GitHub installation token request failed with HTTP ${response.status}.`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
if (typeof result.token !== 'string' || result.token.length === 0) {
|
||||
throw new Error('GitHub returned an empty installation token.');
|
||||
}
|
||||
|
||||
return result.token;
|
||||
}
|
||||
|
||||
function readConfig(environment) {
|
||||
const config = {
|
||||
azureSubscriptionId: environment.AZURE_SUBSCRIPTION_ID,
|
||||
keyVaultName: environment.KEY_VAULT_NAME,
|
||||
keyName: environment.KEY_NAME,
|
||||
githubAppClientId: environment.GITHUB_APP_CLIENT_ID,
|
||||
githubAppInstallationId: environment.GITHUB_APP_INSTALLATION_ID,
|
||||
targetRepository: environment.TARGET_REPOSITORY,
|
||||
};
|
||||
|
||||
if (Object.values(config).some((value) => !value)) {
|
||||
throw new Error('Required GitHub App authentication configuration is missing.');
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const token = await createInstallationToken(readConfig(process.env));
|
||||
process.stdout.write(token);
|
||||
} catch {
|
||||
console.error('GitHub App token generation failed.');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
void main();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
base64ToBase64Url,
|
||||
createInstallationToken,
|
||||
createJwtSigningInput,
|
||||
readConfig,
|
||||
signJwt,
|
||||
};
|
||||
@@ -17,7 +17,7 @@ runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version-file: "python/pyproject.toml"
|
||||
enable-cache: true
|
||||
@@ -32,13 +32,7 @@ runs:
|
||||
if grep -q "name = \"$pkg\"" "$f"; then
|
||||
pkg_dir=$(dirname "$f" | sed 's|python/||')
|
||||
echo "Excluding workspace package: $pkg ($pkg_dir)"
|
||||
if awk '/^\[tool\.uv\.workspace\]/{f=1;next} /^\[/{f=0} f && /^exclude = \[/{found=1} END{exit !found}' python/pyproject.toml; then
|
||||
if ! awk '/^\[tool\.uv\.workspace\]/{f=1;next} /^\[/{f=0} f && /^exclude = \[/ && index($0, "\"'"$pkg_dir"'\"")' python/pyproject.toml | grep -q .; then
|
||||
sed -i.bak '/\[tool\.uv\.workspace\]/,/^\[/ { /^exclude = \[/ s|\]|, "'"$pkg_dir"'"]| }' python/pyproject.toml
|
||||
fi
|
||||
else
|
||||
sed -i.bak '/\[tool\.uv\.workspace\]/a\exclude = ["'"$pkg_dir"'"]' python/pyproject.toml
|
||||
fi
|
||||
sed -i.bak '/\[tool\.uv\.workspace\]/a\exclude = ["'"$pkg_dir"'"]' python/pyproject.toml
|
||||
sed -i.bak '/'"$pkg"' = { workspace = true }/d' python/pyproject.toml
|
||||
fi
|
||||
done
|
||||
@@ -46,4 +40,4 @@ runs:
|
||||
- name: Install the project
|
||||
shell: bash
|
||||
run: |
|
||||
cd python && uv sync --all-packages --all-extras --all-groups --prerelease=if-necessary-or-explicit
|
||||
cd python && uv sync --all-packages --all-extras --dev -U --prerelease=if-necessary-or-explicit
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
name: Save Sample Playbooks
|
||||
description: >
|
||||
Save the cached sample-validation playbooks. Split out from
|
||||
sample-validation-setup (which only restores) so the save runs even when the
|
||||
validation step fails. Combining restore+save via actions/cache would skip the
|
||||
save on a failing job (post-if: success()), so freshly authored playbooks for
|
||||
samples that failed validation would never persist. Invoke this with
|
||||
'if: not-cancelled' after the validation step in each job.
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Save sample playbooks cache
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
# Must match the restore path/key in sample-validation-setup/action.yml and the
|
||||
# sample_validation --playbooks-dir default (samples/sample_validation/playbooks).
|
||||
path: python/samples/sample_validation/playbooks/
|
||||
key: sample-playbooks-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -24,7 +24,7 @@ runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Set up Node.js environment
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
@@ -34,33 +34,17 @@ runs:
|
||||
|
||||
- name: Test Copilot CLI
|
||||
shell: bash
|
||||
run: copilot --version && copilot -p "What can you do in one sentence?"
|
||||
run: copilot -p "What can you do in one sentence?"
|
||||
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ inputs.azure-client-id }}
|
||||
tenant-id: ${{ inputs.azure-tenant-id }}
|
||||
subscription-id: ${{ inputs.azure-subscription-id }}
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
os: ${{ inputs.os }}
|
||||
|
||||
- name: Restore sample playbooks
|
||||
# Restore-only. The matching save is a separate step in each job that runs with
|
||||
# `if: ${{ !cancelled() }}` (see .github/actions/sample-validation-save-playbooks).
|
||||
# A combined actions/cache would skip its post-job save on a failing job
|
||||
# (post-if: success()), so playbooks authored for samples that failed validation
|
||||
# would never persist. Keyed per job so each validate-* job keeps its own playbooks.
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
# Must match the sample_validation --playbooks-dir default, which resolves to
|
||||
# samples/sample_validation/playbooks (see python/scripts/sample_validation/__main__.py).
|
||||
# If a job overrides --playbooks-dir, update this path to match.
|
||||
path: python/samples/sample_validation/playbooks/
|
||||
key: sample-playbooks-${{ github.job }}-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
sample-playbooks-${{ github.job }}-
|
||||
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
with:
|
||||
client-id: ${{ inputs.azure-client-id }}
|
||||
tenant-id: ${{ inputs.azure-tenant-id }}
|
||||
subscription-id: ${{ inputs.azure-subscription-id }}
|
||||
|
||||
@@ -24,14 +24,12 @@ updates:
|
||||
- ".NET"
|
||||
- "dependencies"
|
||||
|
||||
# Maintain dependencies for python.
|
||||
# TODO: Remove these Python Dependabot entries after we have confidence in the
|
||||
# Python dependency-maintenance workflow.
|
||||
# Maintain dependencies for python
|
||||
- package-ecosystem: "pip"
|
||||
directory: "python/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "thursday"
|
||||
day: "monday"
|
||||
labels:
|
||||
- "python"
|
||||
- "dependencies"
|
||||
@@ -39,22 +37,16 @@ updates:
|
||||
directory: "python/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "thursday"
|
||||
day: "monday"
|
||||
labels:
|
||||
- "python"
|
||||
- "dependencies"
|
||||
|
||||
# Maintain dependencies for github-actions
|
||||
- package-ecosystem: "github-actions"
|
||||
# Cover both the standard workflow location and our composite actions.
|
||||
# With `directory: "/"` Dependabot only scans `.github/workflows/*.{yml,yaml}`
|
||||
# plus a root-level `action.yml/action.yaml`. It does NOT recurse into
|
||||
# `.github/actions/*/action.yml`, so the glob below is required to keep the
|
||||
# composite actions in `.github/actions/<name>/` up to date as well.
|
||||
# Ref: https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference#directories-or-directory--
|
||||
directories:
|
||||
- "/"
|
||||
- "/.github/actions/*"
|
||||
# Workflow files stored in the
|
||||
# default location of `.github/workflows`
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "sunday"
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
applyTo: "dotnet/src/Microsoft.Agents.AI.DurableTask/**,dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/**"
|
||||
---
|
||||
|
||||
# Durable Task area code instructions
|
||||
|
||||
The following guidelines apply to pull requests that modify files under
|
||||
`dotnet/src/Microsoft.Agents.AI.DurableTask/**` or
|
||||
`dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/**`:
|
||||
|
||||
## CHANGELOG.md
|
||||
|
||||
- Each pull request that modifies code should add just one bulleted entry to the `CHANGELOG.md` file containing a change title (usually the PR title) and a link to the PR itself.
|
||||
- New PRs should be added to the top of the `CHANGELOG.md` file under a "## [Unreleased]" heading.
|
||||
- If the PR is the first since the last release, the existing "## [Unreleased]" heading should be replaced with a "## v[X.Y.Z]" heading and the PRs since the last release should be added to the new "## [Unreleased]" heading.
|
||||
- The style of new `CHANGELOG.md` entries should match the style of the other entries in the file.
|
||||
- If the PR introduces a breaking change, the changelog entry should be prefixed with "[BREAKING]".
|
||||
@@ -1,43 +1,23 @@
|
||||
### Motivation & Context
|
||||
### Motivation and Context
|
||||
|
||||
<!-- Thank you for your contribution to the Agent Framework repo!
|
||||
Please help reviewers and future users, providing the following information:
|
||||
1. Why is this change required?
|
||||
2. What problem does it solve?
|
||||
3. What scenario does it contribute to?
|
||||
4. If it fixes an open issue, please link to the issue below.
|
||||
4. If it fixes an open issue, please link to the issue here.
|
||||
-->
|
||||
|
||||
### Description & Review Guide
|
||||
### Description
|
||||
|
||||
<!-- Describe your changes, the overall approach, the underlying design.
|
||||
Highlight what you want the reviewers to focus on.
|
||||
These notes will help understanding how your code works. Thanks! -->
|
||||
|
||||
- **What are the major changes?**
|
||||
- **What is the impact of these changes?**
|
||||
- **What do you want reviewers to focus on?**
|
||||
<!-- NOTE FOR AUTOMATED/AI REVIEWERS: the "What do you want reviewers to focus on?"
|
||||
item above is intended for human reviewers only. Automated/AI reviewers should
|
||||
ignore it and review the entire change rather than narrowing scope to it. -->
|
||||
|
||||
|
||||
### Related Issue
|
||||
|
||||
<!-- Which issue does this PR fix? Link it using a GitHub closing keyword so it is
|
||||
closed automatically when this PR is merged, e.g. "Fixes #123" or "Closes #123".
|
||||
PRs that are not linked to an issue may be closed, no matter how valid the change is.
|
||||
Also check whether an open PR already exists for this issue; if so,
|
||||
explain how this PR is different. -->
|
||||
|
||||
Fixes #
|
||||
|
||||
### Contribution Checklist
|
||||
|
||||
<!-- Before submitting this PR, please make sure: -->
|
||||
|
||||
- [ ] The code builds clean without any errors or warnings
|
||||
- [ ] All unit tests pass, and I have added new tests where possible
|
||||
- [ ] The PR follows the [Contribution Guidelines](https://github.com/microsoft/agent-framework/blob/main/CONTRIBUTING.md)
|
||||
- [ ] This PR is linked to an issue and there is no other open PR for this issue (see Related Issue above).
|
||||
- [x] **This is not a breaking change.** If it _is_ a breaking change, add the `breaking change` label (or add "[BREAKING]" to the title prefix, before or after any language prefix) — a workflow keeps the label and title prefix in sync automatically.
|
||||
- [ ] All unit tests pass, and I have added new tests where possible
|
||||
- [ ] **Is this a breaking change?** If yes, add "[BREAKING]" prefix to the title of the PR.
|
||||
@@ -1,76 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
/**
|
||||
* Resolve the issue or pull request author and check their team membership.
|
||||
*
|
||||
* @param {object} opts
|
||||
* @param {object} opts.github - Octokit REST client from actions/github-script
|
||||
* @param {object} opts.context - GitHub Actions context
|
||||
* @param {object} opts.core - GitHub Actions core toolkit
|
||||
* @param {string} opts.teamSlug - Team slug to check membership against
|
||||
* @param {string|number} opts.issueNumber - Issue or pull request number to resolve author for
|
||||
* @param {string} [opts.username] - Explicit user to check instead of the issue or pull request author
|
||||
* @returns {Promise<{author: string|null, isTeamMember: boolean}>}
|
||||
*/
|
||||
async function checkTeamMembership({ github, context, core, teamSlug, issueNumber, username = '' }) {
|
||||
let author = username.trim() || (
|
||||
context.payload.issue?.user?.login ??
|
||||
context.payload.pull_request?.user?.login
|
||||
);
|
||||
|
||||
if (!author) {
|
||||
const number = Number(issueNumber);
|
||||
if (context.payload.pull_request) {
|
||||
const { data: pr } = await github.rest.pulls.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: number,
|
||||
});
|
||||
author = pr.user?.login;
|
||||
} else {
|
||||
const { data: issue } = await github.rest.issues.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: number,
|
||||
});
|
||||
author = issue.user?.login;
|
||||
}
|
||||
}
|
||||
|
||||
if (!author) {
|
||||
core.setFailed('Could not determine issue author (user may be deleted).');
|
||||
return { author: null, isTeamMember: false };
|
||||
}
|
||||
|
||||
try {
|
||||
await github.rest.teams.getByName({
|
||||
org: context.repo.owner,
|
||||
team_slug: teamSlug,
|
||||
});
|
||||
} catch (error) {
|
||||
core.setFailed(`Team lookup failed for ${teamSlug}: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
let isTeamMember = false;
|
||||
try {
|
||||
const teamMembership = await github.rest.teams.getMembershipForUserInOrg({
|
||||
org: context.repo.owner,
|
||||
team_slug: teamSlug,
|
||||
username: author,
|
||||
});
|
||||
isTeamMember = teamMembership.data.state === 'active';
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
core.info(`Author ${author} is not a member of team ${teamSlug}.`);
|
||||
isTeamMember = false;
|
||||
} else {
|
||||
core.setFailed(`Team membership lookup failed for ${author}: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return { author, isTeamMember };
|
||||
}
|
||||
|
||||
module.exports = checkTeamMembership;
|
||||
@@ -1,181 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
function getPullRequest(context) {
|
||||
const pullRequest = context.payload.pull_request;
|
||||
if (!pullRequest?.number || !pullRequest.user?.login) {
|
||||
throw new Error('This script must be run from a pull_request_target event.');
|
||||
}
|
||||
|
||||
return {
|
||||
author: pullRequest.user.login,
|
||||
authorType: pullRequest.user.type,
|
||||
labels: pullRequest.labels?.map((label) => label.name).filter(Boolean) ?? [],
|
||||
number: pullRequest.number,
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureLabel({ github, owner, repo, labelName }) {
|
||||
try {
|
||||
await github.rest.issues.getLabel({
|
||||
owner,
|
||||
repo,
|
||||
name: labelName,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status !== 404) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
try {
|
||||
await github.rest.issues.createLabel({
|
||||
owner,
|
||||
repo,
|
||||
name: labelName,
|
||||
color: 'd93f0b',
|
||||
description: 'Community author has exceeded the open pull request limit.',
|
||||
});
|
||||
} catch (createError) {
|
||||
if (createError.status !== 422) {
|
||||
throw createError;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasLabel(labels, labelName) {
|
||||
if (!labelName) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return labels.some((label) => label.toLowerCase() === labelName.toLowerCase());
|
||||
}
|
||||
|
||||
function isDependabotAuthor({ author, authorType }) {
|
||||
return authorType === 'Bot' && author.toLowerCase() === 'dependabot[bot]';
|
||||
}
|
||||
|
||||
function buildLimitMessage({ author, exemptLabelName, maxOpenPrs, openPrCount }) {
|
||||
return [
|
||||
`Thank you for your contribution, @${author}.`,
|
||||
'',
|
||||
`To keep the review queue manageable, we currently limit community contributors to ${maxOpenPrs} `
|
||||
+ `open pull requests at a time. This PR would put you at ${openPrCount} open pull requests, `
|
||||
+ 'so we are closing it automatically.',
|
||||
'',
|
||||
'Please focus on getting your existing PRs reviewed, merged, or closed before opening another one. '
|
||||
+ `If a maintainer asked you to open this PR, they can apply the \`${exemptLabelName}\` label and reopen it.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
async function getOpenPrCount({ github, owner, repo, author, pullRequestNumber }) {
|
||||
const openPullRequests = await github.paginate(github.rest.pulls.list, {
|
||||
owner,
|
||||
repo,
|
||||
state: 'open',
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
const authorOpenPullRequestNumbers = openPullRequests
|
||||
.filter((pullRequest) => pullRequest.user?.login === author)
|
||||
.map((pullRequest) => pullRequest.number);
|
||||
const currentPrIsOpen = authorOpenPullRequestNumbers.includes(pullRequestNumber);
|
||||
const existingOpenPrCount = currentPrIsOpen
|
||||
? authorOpenPullRequestNumbers.length - 1
|
||||
: authorOpenPullRequestNumbers.length;
|
||||
|
||||
return existingOpenPrCount + 1;
|
||||
}
|
||||
|
||||
async function enforcePrLimit({ github, context, core, exemptLabelName, maxOpenPrs, labelName }) {
|
||||
const { owner, repo } = context.repo;
|
||||
const { author, authorType, labels, number } = getPullRequest(context);
|
||||
|
||||
if (isDependabotAuthor({ author, authorType })) {
|
||||
core.info(`Author ${author} is Dependabot; skipping open PR limit enforcement.`);
|
||||
return {
|
||||
author,
|
||||
closed: false,
|
||||
dependabotExempt: true,
|
||||
openPrCount: null,
|
||||
};
|
||||
}
|
||||
|
||||
if (hasLabel(labels, exemptLabelName)) {
|
||||
core.info(`PR #${number} has the ${exemptLabelName} label; skipping open PR limit enforcement.`);
|
||||
return {
|
||||
author,
|
||||
closed: false,
|
||||
exempt: true,
|
||||
openPrCount: null,
|
||||
};
|
||||
}
|
||||
|
||||
const openPrCount = await getOpenPrCount({
|
||||
github,
|
||||
owner,
|
||||
repo,
|
||||
author,
|
||||
pullRequestNumber: number,
|
||||
});
|
||||
|
||||
if (openPrCount <= maxOpenPrs) {
|
||||
core.info(
|
||||
`${author} has ${openPrCount} open pull request(s), which is within the limit of ${maxOpenPrs}.`,
|
||||
);
|
||||
return {
|
||||
author,
|
||||
closed: false,
|
||||
openPrCount,
|
||||
};
|
||||
}
|
||||
|
||||
await ensureLabel({
|
||||
github,
|
||||
owner,
|
||||
repo,
|
||||
labelName,
|
||||
});
|
||||
|
||||
await github.rest.issues.addLabels({
|
||||
owner,
|
||||
repo,
|
||||
issue_number: number,
|
||||
labels: [labelName],
|
||||
});
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
owner,
|
||||
repo,
|
||||
issue_number: number,
|
||||
body: buildLimitMessage({
|
||||
author,
|
||||
exemptLabelName,
|
||||
maxOpenPrs,
|
||||
openPrCount,
|
||||
}),
|
||||
});
|
||||
|
||||
await github.rest.pulls.update({
|
||||
owner,
|
||||
repo,
|
||||
pull_number: number,
|
||||
state: 'closed',
|
||||
});
|
||||
|
||||
core.info(
|
||||
`${author} has ${openPrCount} open pull request(s), which exceeds the limit of ${maxOpenPrs}. `
|
||||
+ `Closed PR #${number}.`,
|
||||
);
|
||||
|
||||
return {
|
||||
author,
|
||||
closed: true,
|
||||
openPrCount,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildLimitMessage,
|
||||
enforcePrLimit,
|
||||
getOpenPrCount,
|
||||
};
|
||||
@@ -1,212 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
"""Enforce Python package coverage according to package lifecycle."""
|
||||
|
||||
# ruff:file-ignore[print]
|
||||
# ruff:file-ignore[implicit-namespace-package]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET # ruff:ignore[suspicious-xml-etree-import]
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import tomllib
|
||||
|
||||
DEVELOPMENT_STATUS_PREFIX = "Development Status :: "
|
||||
ENFORCED_DEVELOPMENT_STATUS = 4
|
||||
EXEMPT_PACKAGES = {"devui", "lab"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackagePolicy:
|
||||
"""Coverage policy derived from a package's project metadata."""
|
||||
|
||||
directory: str
|
||||
distribution_name: str
|
||||
development_status: int
|
||||
development_status_label: str
|
||||
enforced: bool
|
||||
exempt: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class CoverageStats:
|
||||
"""Line and branch coverage counters."""
|
||||
|
||||
lines_valid: int = 0
|
||||
lines_covered: int = 0
|
||||
branches_valid: int = 0
|
||||
branches_covered: int = 0
|
||||
|
||||
@property
|
||||
def line_coverage_percent(self) -> float:
|
||||
"""Return line coverage as a percentage."""
|
||||
if not self.lines_valid:
|
||||
return 0
|
||||
return self.lines_covered / self.lines_valid * 100
|
||||
|
||||
|
||||
def normalize_coverage_path(path: str) -> str:
|
||||
"""Normalize a coverage path for matching."""
|
||||
return path.replace("\\", "/").lstrip("./")
|
||||
|
||||
|
||||
def load_package_policies(packages_dir: Path) -> list[PackagePolicy]:
|
||||
"""Load lifecycle-based coverage policies from package pyproject files."""
|
||||
policies: list[PackagePolicy] = []
|
||||
for pyproject_path in sorted(packages_dir.glob("*/pyproject.toml")):
|
||||
with pyproject_path.open("rb") as pyproject_file:
|
||||
pyproject = tomllib.load(pyproject_file)
|
||||
|
||||
project = pyproject.get("project", {})
|
||||
distribution_name = str(project.get("name", "")).strip()
|
||||
if not distribution_name:
|
||||
raise ValueError(f"{pyproject_path}: project.name is required")
|
||||
|
||||
status_classifiers = [
|
||||
classifier
|
||||
for classifier in project.get("classifiers", [])
|
||||
if classifier.startswith(DEVELOPMENT_STATUS_PREFIX)
|
||||
]
|
||||
if len(status_classifiers) != 1:
|
||||
raise ValueError(
|
||||
f"{pyproject_path}: expected exactly one Development Status classifier, found {len(status_classifiers)}"
|
||||
)
|
||||
|
||||
match = re.fullmatch(r"Development Status :: (\d+) - (.+)", status_classifiers[0])
|
||||
if match is None:
|
||||
raise ValueError(f"{pyproject_path}: malformed Development Status classifier")
|
||||
|
||||
directory = pyproject_path.parent.name
|
||||
development_status = int(match.group(1))
|
||||
exempt = directory in EXEMPT_PACKAGES
|
||||
policies.append(
|
||||
PackagePolicy(
|
||||
directory=directory,
|
||||
distribution_name=distribution_name,
|
||||
development_status=development_status,
|
||||
development_status_label=match.group(2),
|
||||
enforced=development_status >= ENFORCED_DEVELOPMENT_STATUS and not exempt,
|
||||
exempt=exempt,
|
||||
)
|
||||
)
|
||||
|
||||
if not policies:
|
||||
raise ValueError(f"No package pyproject.toml files found below {packages_dir}")
|
||||
return policies
|
||||
|
||||
|
||||
def parse_coverage_xml(xml_path: Path) -> tuple[dict[str, CoverageStats], float, float]:
|
||||
"""Parse Cobertura XML and aggregate coverage by package directory."""
|
||||
root = ET.parse(xml_path).getroot() # ruff:ignore[suspicious-xml-element-tree-usage] # Trusted CI-generated coverage report.
|
||||
package_stats: dict[str, CoverageStats] = {}
|
||||
|
||||
for class_elem in root.findall(".//class"):
|
||||
file_path = normalize_coverage_path(class_elem.get("filename", ""))
|
||||
path_parts = file_path.split("/")
|
||||
try:
|
||||
packages_index = path_parts.index("packages")
|
||||
package_directory = path_parts[packages_index + 1]
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
stats = package_stats.setdefault(package_directory, CoverageStats())
|
||||
for line in class_elem.findall(".//line"):
|
||||
stats.lines_valid += 1
|
||||
if int(line.get("hits", 0)) > 0:
|
||||
stats.lines_covered += 1
|
||||
|
||||
if line.get("branch") != "true":
|
||||
continue
|
||||
condition_coverage = line.get("condition-coverage", "")
|
||||
match = re.search(r"\((\d+)/(\d+)\)", condition_coverage)
|
||||
if match is not None:
|
||||
stats.branches_covered += int(match.group(1))
|
||||
stats.branches_valid += int(match.group(2))
|
||||
|
||||
return (
|
||||
package_stats,
|
||||
float(root.get("line-rate", 0)) * 100,
|
||||
float(root.get("branch-rate", 0)) * 100,
|
||||
)
|
||||
|
||||
|
||||
def check_coverage(xml_path: Path, threshold: float, packages_dir: Path) -> bool:
|
||||
"""Check all lifecycle-enforced packages against the coverage threshold."""
|
||||
policies = load_package_policies(packages_dir)
|
||||
package_stats, overall_line_coverage, overall_branch_coverage = parse_coverage_xml(xml_path)
|
||||
|
||||
print("\n" + "=" * 110)
|
||||
print("PYTHON PACKAGE TEST COVERAGE")
|
||||
print("=" * 110)
|
||||
print(f"Overall Line Coverage: {overall_line_coverage:.1f}%")
|
||||
print(f"Overall Branch Coverage: {overall_branch_coverage:.1f}%")
|
||||
print(f"Enforced Threshold: {threshold:.1f}%")
|
||||
print("-" * 110)
|
||||
print(f"{'Package':<48} {'Stage':<20} {'Policy':<14} {'Lines':<12} {'Line Cov':<10}")
|
||||
print("-" * 110)
|
||||
|
||||
failed_packages: list[str] = []
|
||||
for policy in sorted(policies, key=lambda item: (not item.enforced, item.distribution_name)):
|
||||
stats = package_stats.get(policy.directory)
|
||||
if policy.exempt:
|
||||
policy_label = "EXEMPT"
|
||||
elif policy.enforced:
|
||||
policy_label = "ENFORCED"
|
||||
else:
|
||||
policy_label = "REPORT ONLY"
|
||||
|
||||
if stats is None:
|
||||
lines = "-"
|
||||
coverage = "missing"
|
||||
if policy.enforced:
|
||||
failed_packages.append(f"{policy.distribution_name} (missing from coverage report)")
|
||||
else:
|
||||
lines = f"{stats.lines_covered}/{stats.lines_valid}"
|
||||
coverage = f"{stats.line_coverage_percent:.1f}%"
|
||||
if policy.enforced and stats.line_coverage_percent < threshold:
|
||||
failed_packages.append(f"{policy.distribution_name} ({coverage})")
|
||||
|
||||
stage = f"{policy.development_status} - {policy.development_status_label}"
|
||||
print(f"{policy.distribution_name:<48} {stage:<20} {policy_label:<14} {lines:<12} {coverage:<10}")
|
||||
|
||||
print("-" * 110)
|
||||
if failed_packages:
|
||||
print(f"\nFAILED: Enforced packages below {threshold:.1f}% or missing:")
|
||||
for package in failed_packages:
|
||||
print(f" - {package}")
|
||||
return False
|
||||
|
||||
print(f"\nPASSED: All non-exempt Beta-or-higher packages meet {threshold:.1f}% line coverage.")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run the coverage policy check."""
|
||||
if len(sys.argv) != 3:
|
||||
print(f"Usage: {sys.argv[0]} <coverage-xml-path> <threshold>")
|
||||
return 1
|
||||
|
||||
try:
|
||||
threshold = float(sys.argv[2])
|
||||
except ValueError:
|
||||
print(f"Error: Invalid threshold value: {sys.argv[2]}")
|
||||
return 1
|
||||
|
||||
repository_root = Path(__file__).resolve().parents[2]
|
||||
try:
|
||||
passed = check_coverage(
|
||||
Path(sys.argv[1]),
|
||||
threshold,
|
||||
repository_root / "python" / "packages",
|
||||
)
|
||||
except (FileNotFoundError, ET.ParseError, ValueError) as error:
|
||||
print(f"Error: {error}")
|
||||
return 1
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,170 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
const DECISIVE_REVIEW_STATES = new Set(['APPROVED', 'CHANGES_REQUESTED', 'DISMISSED']);
|
||||
const SHA_PATTERN = /^[0-9a-f]{40}$/;
|
||||
const BRANCH_PATTERN = /^[a-zA-Z0-9_./-]+$/;
|
||||
|
||||
function assertValidSha(sha, description) {
|
||||
if (!SHA_PATTERN.test(sha)) {
|
||||
throw new Error(`GitHub returned an invalid ${description} SHA.`);
|
||||
}
|
||||
}
|
||||
|
||||
function hasWritePermission(permissionData) {
|
||||
return permissionData.user?.permissions?.push === true
|
||||
|| ['admin', 'maintain', 'write'].includes(permissionData.permission);
|
||||
}
|
||||
|
||||
function latestDecisiveReviews(reviews) {
|
||||
const latestByReviewer = new Map();
|
||||
const sortedReviews = [...reviews].sort((left, right) => {
|
||||
const submittedComparison = (left.submitted_at || '').localeCompare(right.submitted_at || '');
|
||||
return submittedComparison || Number(left.id) - Number(right.id);
|
||||
});
|
||||
|
||||
for (const review of sortedReviews) {
|
||||
const state = review.state?.toUpperCase();
|
||||
const reviewer = review.user?.login?.toLowerCase();
|
||||
if (reviewer && DECISIVE_REVIEW_STATES.has(state)) {
|
||||
latestByReviewer.set(reviewer, review);
|
||||
}
|
||||
}
|
||||
|
||||
return latestByReviewer;
|
||||
}
|
||||
|
||||
async function resolvePullRequest({ github, context, core, prNumber, requiredApprovals }) {
|
||||
if (!/^[0-9]+$/.test(prNumber)) {
|
||||
throw new Error('Invalid PR number. Only numeric values are allowed.');
|
||||
}
|
||||
|
||||
const pullNumber = Number(prNumber);
|
||||
const { data: pullRequest } = await github.rest.pulls.get({
|
||||
...context.repo,
|
||||
pull_number: pullNumber,
|
||||
});
|
||||
|
||||
if (pullRequest.state !== 'open') {
|
||||
throw new Error(`PR #${pullNumber} is not open (state: ${pullRequest.state}).`);
|
||||
}
|
||||
|
||||
const headSha = pullRequest.head.sha;
|
||||
const baseSha = pullRequest.base.sha;
|
||||
assertValidSha(headSha, 'PR head');
|
||||
assertValidSha(baseSha, 'PR base');
|
||||
|
||||
const reviews = await github.paginate(github.rest.pulls.listReviews, {
|
||||
...context.repo,
|
||||
pull_number: pullNumber,
|
||||
per_page: 100,
|
||||
});
|
||||
const latestReviews = latestDecisiveReviews(reviews);
|
||||
const author = pullRequest.user?.login?.toLowerCase();
|
||||
const approvalCandidates = [...latestReviews.entries()]
|
||||
.filter(([, review]) => review.state.toUpperCase() === 'APPROVED')
|
||||
.filter(([, review]) => review.commit_id === headSha)
|
||||
.filter(([reviewer]) => reviewer !== author);
|
||||
|
||||
const approvedMaintainers = [];
|
||||
for (const [reviewer] of approvalCandidates) {
|
||||
const { data: permissionData } = await github.rest.repos.getCollaboratorPermissionLevel({
|
||||
...context.repo,
|
||||
username: reviewer,
|
||||
});
|
||||
if (hasWritePermission(permissionData)) {
|
||||
approvedMaintainers.push(reviewer);
|
||||
} else {
|
||||
core.info(`Ignoring approval from ${reviewer}: reviewer does not have write permission.`);
|
||||
}
|
||||
}
|
||||
|
||||
if (approvedMaintainers.length < requiredApprovals) {
|
||||
throw new Error(
|
||||
`PR #${pullNumber} head ${headSha} requires ${requiredApprovals} approvals from unique `
|
||||
+ `write-capable maintainers; found ${approvedMaintainers.length}.`,
|
||||
);
|
||||
}
|
||||
|
||||
core.info(
|
||||
`PR #${pullNumber} head ${headSha} approved by: ${approvedMaintainers.join(', ')}.`,
|
||||
);
|
||||
return {
|
||||
baseRef: baseSha,
|
||||
checkoutRef: headSha,
|
||||
description: `PR #${pullNumber}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveBranch({ github, context, core, branch }) {
|
||||
if (!BRANCH_PATTERN.test(branch)) {
|
||||
throw new Error(
|
||||
'Invalid branch name. Only alphanumeric characters, hyphens, underscores, dots, and slashes '
|
||||
+ 'are allowed.',
|
||||
);
|
||||
}
|
||||
|
||||
const [{ data: repository }, { data: targetBranch }] = await Promise.all([
|
||||
github.rest.repos.get(context.repo),
|
||||
github.rest.repos.getBranch({ ...context.repo, branch }),
|
||||
]);
|
||||
const { data: baseBranch } = await github.rest.repos.getBranch({
|
||||
...context.repo,
|
||||
branch: repository.default_branch,
|
||||
});
|
||||
|
||||
const checkoutRef = targetBranch.commit.sha;
|
||||
const baseRef = baseBranch.commit.sha;
|
||||
assertValidSha(checkoutRef, 'branch head');
|
||||
assertValidSha(baseRef, 'default branch');
|
||||
core.info(`Branch ${branch} resolved to immutable commit ${checkoutRef}.`);
|
||||
|
||||
return {
|
||||
baseRef,
|
||||
checkoutRef,
|
||||
description: `branch ${branch}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a manually requested integration-test target to an immutable commit.
|
||||
*
|
||||
* Pull requests must have fresh approvals from two unique write-capable
|
||||
* maintainers for the exact head commit. Branches are limited to branches in
|
||||
* the base repository and are pinned to their current commit.
|
||||
*/
|
||||
async function resolveIntegrationTestTarget({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
prNumber = '',
|
||||
branch = '',
|
||||
requiredApprovals = 2,
|
||||
}) {
|
||||
const normalizedPrNumber = prNumber.trim();
|
||||
const normalizedBranch = branch.trim();
|
||||
|
||||
if (normalizedPrNumber && normalizedBranch) {
|
||||
throw new Error('Please provide either a PR number or a branch name, not both.');
|
||||
}
|
||||
if (!normalizedPrNumber && !normalizedBranch) {
|
||||
throw new Error('Please provide either a PR number or a branch name.');
|
||||
}
|
||||
|
||||
if (normalizedPrNumber) {
|
||||
return resolvePullRequest({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
prNumber: normalizedPrNumber,
|
||||
requiredApprovals,
|
||||
});
|
||||
}
|
||||
return resolveBranch({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
branch: normalizedBranch,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = resolveIntegrationTestTarget;
|
||||
@@ -1,253 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
const BREAKING_CHANGE_LABEL = 'breaking change';
|
||||
const BREAKING_PREFIX = '[BREAKING]';
|
||||
|
||||
const DEFAULT_PREFIX_LABELS = Object.freeze({
|
||||
python: 'Python',
|
||||
'.NET': '.NET',
|
||||
});
|
||||
|
||||
const DEFAULT_BRACKET_PREFIX_LABELS = Object.freeze({
|
||||
[BREAKING_CHANGE_LABEL]: BREAKING_PREFIX,
|
||||
});
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function getMatchingValueByKey(valuesByKey, keyToFind) {
|
||||
const matchingKey = Object.keys(valuesByKey).find((key) => key.toLowerCase() === keyToFind.toLowerCase());
|
||||
return matchingKey === undefined ? null : valuesByKey[matchingKey];
|
||||
}
|
||||
|
||||
function getPrefixPattern(prefixes) {
|
||||
return prefixes.map(escapeRegExp).join('|');
|
||||
}
|
||||
|
||||
function canonicalizePrefix(prefix, prefixes) {
|
||||
return prefixes.find((knownPrefix) => knownPrefix.toLowerCase() === prefix.toLowerCase()) ?? prefix;
|
||||
}
|
||||
|
||||
function normalizeLeadingBracketPrefix(title, bracketPrefixes) {
|
||||
const bracketPattern = getPrefixPattern(bracketPrefixes);
|
||||
if (!bracketPattern) {
|
||||
return title;
|
||||
}
|
||||
|
||||
const leadingBracketPrefix = new RegExp(`^(${bracketPattern})(?=\\s|$)`, 'i');
|
||||
return title.replace(
|
||||
leadingBracketPrefix,
|
||||
(bracketPrefix) => canonicalizePrefix(bracketPrefix, bracketPrefixes),
|
||||
);
|
||||
}
|
||||
|
||||
function parseLeadingTitlePrefix(title, titlePrefixes) {
|
||||
const titlePrefixPattern = getPrefixPattern(titlePrefixes);
|
||||
if (!titlePrefixPattern) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const match = title.match(new RegExp(`^(${titlePrefixPattern}):\\s*`, 'i'));
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
prefix: canonicalizePrefix(match[1], titlePrefixes),
|
||||
rest: title.slice(match[0].length).trimStart(),
|
||||
};
|
||||
}
|
||||
|
||||
function removeBracketPrefixToken(title, bracketPrefix) {
|
||||
const bracketPrefixPattern = escapeRegExp(bracketPrefix);
|
||||
return title
|
||||
.replace(new RegExp(`(^|\\s+)${bracketPrefixPattern}(?=\\s|$)`, 'ig'), '$1')
|
||||
.replace(/\s{2,}/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function addTitlePrefix(title, prefix, bracketPrefixes = Object.values(DEFAULT_BRACKET_PREFIX_LABELS)) {
|
||||
const bracketPattern = getPrefixPattern(bracketPrefixes);
|
||||
const prefixPattern = escapeRegExp(prefix);
|
||||
|
||||
if (bracketPattern) {
|
||||
const bracketThenTitlePrefix = new RegExp(`^(${bracketPattern})(\\s+)(${prefixPattern})(?=:)`, 'i');
|
||||
if (bracketThenTitlePrefix.test(title)) {
|
||||
return title.replace(
|
||||
bracketThenTitlePrefix,
|
||||
(match, bracketPrefix, spacing) => `${canonicalizePrefix(bracketPrefix, bracketPrefixes)}${spacing}${prefix}`,
|
||||
);
|
||||
}
|
||||
|
||||
title = normalizeLeadingBracketPrefix(title, bracketPrefixes);
|
||||
}
|
||||
|
||||
if (!title.startsWith(`${prefix}: `)) {
|
||||
const existingTitlePrefix = new RegExp(`^${prefixPattern}:\\s*`, 'i');
|
||||
if (existingTitlePrefix.test(title)) {
|
||||
return title.replace(existingTitlePrefix, `${prefix}: `);
|
||||
}
|
||||
|
||||
return `${prefix}: ${title}`;
|
||||
}
|
||||
|
||||
return title;
|
||||
}
|
||||
|
||||
function hasBracketPrefix(title, bracketPrefix, titlePrefixes = Object.values(DEFAULT_PREFIX_LABELS)) {
|
||||
const bracketPrefixPattern = escapeRegExp(bracketPrefix);
|
||||
const leadingBracketPrefix = new RegExp(`^${bracketPrefixPattern}(?=\\s|$)`, 'i');
|
||||
if (leadingBracketPrefix.test(title)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const leadingTitlePrefix = parseLeadingTitlePrefix(title, titlePrefixes);
|
||||
if (!leadingTitlePrefix) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return leadingBracketPrefix.test(leadingTitlePrefix.rest);
|
||||
}
|
||||
|
||||
function addBracketPrefix(title, bracketPrefix, titlePrefixes = Object.values(DEFAULT_PREFIX_LABELS)) {
|
||||
const bracketPrefixPattern = escapeRegExp(bracketPrefix);
|
||||
const leadingBracketPrefix = new RegExp(`^${bracketPrefixPattern}(?=\\s|$)`, 'i');
|
||||
if (leadingBracketPrefix.test(title)) {
|
||||
return title.replace(leadingBracketPrefix, bracketPrefix);
|
||||
}
|
||||
|
||||
const leadingTitlePrefix = parseLeadingTitlePrefix(title, titlePrefixes);
|
||||
if (leadingTitlePrefix) {
|
||||
if (leadingBracketPrefix.test(leadingTitlePrefix.rest)) {
|
||||
const normalizedRest = leadingTitlePrefix.rest.replace(leadingBracketPrefix, bracketPrefix);
|
||||
return `${leadingTitlePrefix.prefix}: ${normalizedRest}`;
|
||||
}
|
||||
|
||||
const titleWithoutBracketPrefix = removeBracketPrefixToken(leadingTitlePrefix.rest, bracketPrefix);
|
||||
return `${leadingTitlePrefix.prefix}: ${bracketPrefix}`
|
||||
+ (titleWithoutBracketPrefix ? ` ${titleWithoutBracketPrefix}` : '');
|
||||
}
|
||||
|
||||
const titleWithoutBracketPrefix = removeBracketPrefixToken(title, bracketPrefix);
|
||||
return `${bracketPrefix}${titleWithoutBracketPrefix ? ` ${titleWithoutBracketPrefix}` : ''}`;
|
||||
}
|
||||
|
||||
function hasLabel(labels, labelName) {
|
||||
return labels.some((label) => label.toLowerCase() === labelName.toLowerCase());
|
||||
}
|
||||
|
||||
function getCurrentTitle(context) {
|
||||
switch (context.eventName) {
|
||||
case 'issues':
|
||||
return context.payload.issue.title;
|
||||
case 'pull_request_target':
|
||||
return context.payload.pull_request.title;
|
||||
default:
|
||||
throw new Error(`Unrecognized eventName: ${context.eventName}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function updateTitleForAddedLabel({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
prefixLabels = DEFAULT_PREFIX_LABELS,
|
||||
bracketPrefixLabels = DEFAULT_BRACKET_PREFIX_LABELS,
|
||||
}) {
|
||||
const labelAdded = context.payload.label?.name;
|
||||
if (!labelAdded) {
|
||||
throw new Error('This script must be run from a labeled event.');
|
||||
}
|
||||
|
||||
const currentTitle = getCurrentTitle(context);
|
||||
let newTitle = null;
|
||||
|
||||
const titlePrefix = getMatchingValueByKey(prefixLabels, labelAdded);
|
||||
if (titlePrefix !== null) {
|
||||
newTitle = addTitlePrefix(currentTitle, titlePrefix, Object.values(bracketPrefixLabels));
|
||||
}
|
||||
|
||||
const bracketPrefix = getMatchingValueByKey(bracketPrefixLabels, labelAdded);
|
||||
if (bracketPrefix !== null) {
|
||||
newTitle = addBracketPrefix(currentTitle, bracketPrefix, Object.values(prefixLabels));
|
||||
}
|
||||
|
||||
if (newTitle === null) {
|
||||
core.info(`No title prefix configured for label "${labelAdded}".`);
|
||||
return { updated: false, newTitle: currentTitle };
|
||||
}
|
||||
|
||||
if (newTitle === currentTitle) {
|
||||
core.info(`Title already includes the prefix for label "${labelAdded}".`);
|
||||
return { updated: false, newTitle };
|
||||
}
|
||||
|
||||
switch (context.eventName) {
|
||||
case 'issues':
|
||||
await github.rest.issues.update({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: newTitle,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'pull_request_target':
|
||||
await github.rest.pulls.update({
|
||||
pull_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: newTitle,
|
||||
});
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new Error(`Unrecognized eventName: ${context.eventName}`);
|
||||
}
|
||||
|
||||
return { updated: true, newTitle };
|
||||
}
|
||||
|
||||
async function syncBreakingChangeLabelFromTitle({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
labelName = BREAKING_CHANGE_LABEL,
|
||||
bracketPrefix = BREAKING_PREFIX,
|
||||
titlePrefixes = Object.values(DEFAULT_PREFIX_LABELS),
|
||||
}) {
|
||||
const pullRequest = context.payload.pull_request;
|
||||
if (!pullRequest) {
|
||||
throw new Error('This script must be run from a pull_request_target event.');
|
||||
}
|
||||
|
||||
const title = pullRequest.title || '';
|
||||
if (!hasBracketPrefix(title, bracketPrefix, titlePrefixes)) {
|
||||
core.info(`Title does not include ${bracketPrefix} in the title prefix.`);
|
||||
return { added: false };
|
||||
}
|
||||
|
||||
const labels = pullRequest.labels?.map((label) => label.name).filter(Boolean) ?? [];
|
||||
if (hasLabel(labels, labelName)) {
|
||||
core.info(`PR already has the "${labelName}" label.`);
|
||||
return { added: false };
|
||||
}
|
||||
|
||||
await github.rest.issues.addLabels({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
labels: [labelName],
|
||||
});
|
||||
|
||||
return { added: true };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
addBracketPrefix,
|
||||
addTitlePrefix,
|
||||
hasBracketPrefix,
|
||||
syncBreakingChangeLabelFromTitle,
|
||||
updateTitleForAddedLabel,
|
||||
};
|
||||
@@ -1,116 +0,0 @@
|
||||
---
|
||||
name: pull-requests
|
||||
description: >
|
||||
Guidance for creating pull requests and handling PR review comments in the
|
||||
Agent Framework repository. Use this when writing a PR description (filling out
|
||||
the PR template) or when responding to and resolving review comments on an
|
||||
existing PR.
|
||||
---
|
||||
|
||||
# Pull Request Workflow
|
||||
|
||||
This skill covers two tasks: (1) writing a high-quality PR description, and
|
||||
(2) handling review comments on an existing PR.
|
||||
|
||||
## 1. Writing the PR description
|
||||
|
||||
Always follow the repository PR template at
|
||||
[`.github/pull_request_template.md`](../../pull_request_template.md). Keep its
|
||||
exact structure and headings. Fill every section:
|
||||
|
||||
### `### Motivation & Context`
|
||||
Explain *why* the change is needed: the problem it solves and the scenario it
|
||||
contributes to. Describe the net change relative to `main` — this is implied, so
|
||||
do **not** spell out "vs main" explicitly.
|
||||
|
||||
### `### Description & Review Guide`
|
||||
Describe the changes, the overall approach, and the design. Answer the three
|
||||
prompts:
|
||||
- **What are the major changes?**
|
||||
- **What is the impact of these changes?**
|
||||
- **What do you want reviewers to focus on?** — This item is for **human
|
||||
reviewers only**. Automated/AI reviewers must ignore it and review the entire
|
||||
change rather than narrowing scope to it.
|
||||
|
||||
### `### Related Issue`
|
||||
Link the issue the PR fixes using a GitHub closing keyword (`Fixes #123` /
|
||||
`Closes #123`) so it closes automatically on merge. A PR with no linked issue may
|
||||
be closed regardless of how valid the change is. Before opening, confirm there is
|
||||
no other open PR for the same issue; if there is, explain how this PR differs.
|
||||
|
||||
### `### Contribution Checklist`
|
||||
Check every item that applies. For the breaking-change item:
|
||||
- Leave **"This is not a breaking change."** checked for the common case.
|
||||
- If the change **is** breaking, add the `breaking change` label **or** put
|
||||
`[BREAKING]` in the title prefix, before or after a language prefix such as
|
||||
`Python:` or `.NET:` — workflows keep the label and the title prefix in sync
|
||||
automatically (see `.github/workflows/label-title-prefix.yml` and
|
||||
`.github/workflows/label-pr.yml`).
|
||||
|
||||
### Do not
|
||||
- Do **not** add ad-hoc sections such as "Validation" or "Tests run"; CI/CD and
|
||||
the checklist already cover validation status.
|
||||
- Do **not** remove or reorder the template's headings.
|
||||
|
||||
### Creating the PR
|
||||
Open new PRs as **drafts** until they are ready for review. Example:
|
||||
|
||||
```bash
|
||||
gh pr create --repo microsoft/agent-framework --base main \
|
||||
--head <your-fork-owner>:<branch> --draft \
|
||||
--title "<concise title>" --body "<body following the template>"
|
||||
```
|
||||
|
||||
## 2. Handling review comments
|
||||
|
||||
When a PR receives review comments, follow this sequence — **do not start editing
|
||||
code before the user has reviewed the plan**:
|
||||
|
||||
1. **Review the comments.** Read every review comment and thread on the PR,
|
||||
including inline code comments and general review summaries.
|
||||
2. **Make a plan.** Produce a concrete plan describing how each comment will be
|
||||
addressed (or why it should not be, with reasoning).
|
||||
3. **Let the user review the plan.** Present the plan and wait for the user's
|
||||
approval or adjustments before implementing anything.
|
||||
4. **Implement.** Make the agreed changes.
|
||||
5. **Reply to every comment.** Add a reply to **all** comments explaining how it
|
||||
was addressed (or the agreed outcome) — leave none unanswered.
|
||||
6. **Resolve resolved threads.** Mark a review thread as resolved only when the
|
||||
comment has actually been addressed.
|
||||
|
||||
### Useful commands
|
||||
|
||||
List review comments and threads:
|
||||
|
||||
```bash
|
||||
# Inline review comments
|
||||
gh api repos/{owner}/{repo}/pulls/{pr}/comments
|
||||
|
||||
# Review threads with resolution state (GraphQL)
|
||||
gh api graphql -f query='
|
||||
query($owner:String!,$repo:String!,$pr:Int!){
|
||||
repository(owner:$owner,name:$repo){
|
||||
pullRequest(number:$pr){
|
||||
reviewThreads(first:100){
|
||||
nodes{ id isResolved comments(first:50){ nodes{ id body author{login} } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}' -F owner={owner} -F repo={repo} -F pr={pr}
|
||||
```
|
||||
|
||||
Reply to an inline review comment:
|
||||
|
||||
```bash
|
||||
gh api repos/{owner}/{repo}/pulls/{pr}/comments/{comment_id}/replies \
|
||||
-f body="Addressed in <commit>: <explanation>"
|
||||
```
|
||||
|
||||
Resolve a review thread (needs the thread node id from the GraphQL query above):
|
||||
|
||||
```bash
|
||||
gh api graphql -f query='
|
||||
mutation($threadId:ID!){
|
||||
resolveReviewThread(input:{threadId:$threadId}){ thread{ isResolved } }
|
||||
}' -F threadId={thread_id}
|
||||
```
|
||||
@@ -1,249 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
/**
|
||||
* Tests for check_team_membership.js.
|
||||
*
|
||||
* Run with: node --test .github/tests/test_check_team_membership.js
|
||||
*/
|
||||
|
||||
const { describe, it } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const checkTeamMembership = require('../scripts/check_team_membership.js');
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createMocks({
|
||||
payloadIssue = undefined,
|
||||
payloadPullRequest = undefined,
|
||||
apiUser = 'api-user',
|
||||
teamState = 'active',
|
||||
} = {}) {
|
||||
const core = {
|
||||
_infoMessages: [],
|
||||
_failedMessages: [],
|
||||
info(msg) { this._infoMessages.push(msg); },
|
||||
setFailed(msg) { this._failedMessages.push(msg); },
|
||||
};
|
||||
|
||||
const payload = {};
|
||||
if (payloadIssue !== undefined) {
|
||||
payload.issue = payloadIssue;
|
||||
}
|
||||
if (payloadPullRequest !== undefined) {
|
||||
payload.pull_request = payloadPullRequest;
|
||||
}
|
||||
|
||||
const context = {
|
||||
payload,
|
||||
repo: { owner: 'test-org', repo: 'test-repo' },
|
||||
};
|
||||
|
||||
const github = {
|
||||
rest: {
|
||||
issues: {
|
||||
get: async () => ({
|
||||
data: { user: apiUser ? { login: apiUser } : null },
|
||||
}),
|
||||
},
|
||||
pulls: {
|
||||
get: async () => ({
|
||||
data: { user: apiUser ? { login: apiUser } : null },
|
||||
}),
|
||||
},
|
||||
teams: {
|
||||
getByName: async () => ({}),
|
||||
getMembershipForUserInOrg: async () => ({
|
||||
data: { state: teamState },
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return { core, context, github };
|
||||
}
|
||||
|
||||
const BASE_OPTS = { teamSlug: 'my-team', issueNumber: '123' };
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Author resolution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('author resolution', () => {
|
||||
it('uses an explicit username instead of the issue author', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'issue-author' } },
|
||||
});
|
||||
let issuesGetCalled = false;
|
||||
github.rest.issues.get = async () => {
|
||||
issuesGetCalled = true;
|
||||
return { data: { user: { login: 'api-user' } } };
|
||||
};
|
||||
|
||||
const result = await checkTeamMembership({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
...BASE_OPTS,
|
||||
username: 'comment-author',
|
||||
});
|
||||
|
||||
assert.equal(result.author, 'comment-author');
|
||||
assert.equal(issuesGetCalled, false);
|
||||
});
|
||||
|
||||
it('resolves author from event payload', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'payload-user' } },
|
||||
});
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.author, 'payload-user');
|
||||
});
|
||||
|
||||
it('resolves author from pull_request event payload', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadPullRequest: { user: { login: 'pr-author' } },
|
||||
});
|
||||
let issuesGetCalled = false;
|
||||
github.rest.issues.get = async () => {
|
||||
issuesGetCalled = true;
|
||||
return { data: { user: { login: 'api-user' } } };
|
||||
};
|
||||
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.author, 'pr-author');
|
||||
assert.equal(issuesGetCalled, false);
|
||||
});
|
||||
|
||||
it('resolves author via pulls API when pull_request payload user is null', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadPullRequest: { user: null },
|
||||
apiUser: 'fetched-pr-author',
|
||||
});
|
||||
let pullsGetCalled = false;
|
||||
github.rest.pulls.get = async () => {
|
||||
pullsGetCalled = true;
|
||||
return { data: { user: { login: 'fetched-pr-author' } } };
|
||||
};
|
||||
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.author, 'fetched-pr-author');
|
||||
assert.equal(pullsGetCalled, true);
|
||||
});
|
||||
|
||||
it('resolves author via API when payload issue is absent', async () => {
|
||||
const { github, context, core } = createMocks({ apiUser: 'api-user' });
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.author, 'api-user');
|
||||
});
|
||||
|
||||
it('resolves author via API when payload issue user is null (deleted account)', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: null },
|
||||
apiUser: 'fetched-user',
|
||||
});
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.author, 'fetched-user');
|
||||
});
|
||||
|
||||
it('handles deleted account when API also returns null user', async () => {
|
||||
const { github, context, core } = createMocks({ apiUser: null });
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.author, null);
|
||||
assert.equal(result.isTeamMember, false);
|
||||
assert.ok(core._failedMessages.some(m => m.includes('deleted')));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Team lookup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('team lookup', () => {
|
||||
it('fails the job when team lookup errors', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'user1' } },
|
||||
});
|
||||
const error = new Error('Bad credentials');
|
||||
github.rest.teams.getByName = async () => { throw error; };
|
||||
|
||||
await assert.rejects(
|
||||
() => checkTeamMembership({ github, context, core, ...BASE_OPTS }),
|
||||
(err) => err === error,
|
||||
);
|
||||
assert.ok(core._failedMessages.some(m => m.includes('Team lookup failed')));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Team membership
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('team membership', () => {
|
||||
it('returns true for active team member', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'member' } },
|
||||
teamState: 'active',
|
||||
});
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.isTeamMember, true);
|
||||
});
|
||||
|
||||
it('returns false for pending team member', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'pending-user' } },
|
||||
teamState: 'pending',
|
||||
});
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.isTeamMember, false);
|
||||
});
|
||||
|
||||
it('treats 404 membership response as non-member without failing', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'outsider' } },
|
||||
});
|
||||
const notFoundError = new Error('Not Found');
|
||||
notFoundError.status = 404;
|
||||
github.rest.teams.getMembershipForUserInOrg = async () => { throw notFoundError; };
|
||||
|
||||
const result = await checkTeamMembership({ github, context, core, ...BASE_OPTS });
|
||||
assert.equal(result.isTeamMember, false);
|
||||
assert.equal(core._failedMessages.length, 0);
|
||||
assert.ok(core._infoMessages.some(m => m.includes('not a member')));
|
||||
});
|
||||
|
||||
it('fails the job on non-404 membership errors', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'user1' } },
|
||||
});
|
||||
const serverError = new Error('Internal Server Error');
|
||||
serverError.status = 500;
|
||||
github.rest.teams.getMembershipForUserInOrg = async () => { throw serverError; };
|
||||
|
||||
await assert.rejects(
|
||||
() => checkTeamMembership({ github, context, core, ...BASE_OPTS }),
|
||||
(err) => err === serverError,
|
||||
);
|
||||
assert.ok(core._failedMessages.some(m => m.includes('membership lookup failed')));
|
||||
});
|
||||
|
||||
it('fails the job on membership errors without status code', async () => {
|
||||
const { github, context, core } = createMocks({
|
||||
payloadIssue: { user: { login: 'user1' } },
|
||||
});
|
||||
const networkError = new Error('ECONNREFUSED');
|
||||
github.rest.teams.getMembershipForUserInOrg = async () => { throw networkError; };
|
||||
|
||||
await assert.rejects(
|
||||
() => checkTeamMembership({ github, context, core, ...BASE_OPTS }),
|
||||
(err) => err === networkError,
|
||||
);
|
||||
assert.ok(core._failedMessages.some(m => m.includes('membership lookup failed')));
|
||||
});
|
||||
});
|
||||
@@ -1,125 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
const { describe, it } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const {
|
||||
base64ToBase64Url,
|
||||
createInstallationToken,
|
||||
createJwtSigningInput,
|
||||
readConfig,
|
||||
} = require('../actions/github-app-token/create-token.js');
|
||||
|
||||
const CONFIG = {
|
||||
azureSubscriptionId: 'subscription-id',
|
||||
keyVaultName: 'vault-name',
|
||||
keyName: 'key-name',
|
||||
githubAppClientId: 'client-id',
|
||||
githubAppInstallationId: '12345',
|
||||
targetRepository: 'microsoft/agent-framework',
|
||||
};
|
||||
|
||||
describe('GitHub App token creation', () => {
|
||||
it('creates a short-lived GitHub App JWT', () => {
|
||||
const signingInput = createJwtSigningInput('client-id', 1_000);
|
||||
const [encodedHeader, encodedPayload] = signingInput.split('.');
|
||||
const header = JSON.parse(Buffer.from(encodedHeader, 'base64url').toString());
|
||||
const payload = JSON.parse(Buffer.from(encodedPayload, 'base64url').toString());
|
||||
|
||||
assert.deepEqual(header, { alg: 'RS256', typ: 'JWT' });
|
||||
assert.deepEqual(payload, { iat: 940, exp: 1_540, iss: 'client-id' });
|
||||
});
|
||||
|
||||
it('converts Key Vault signatures to unpadded base64url', () => {
|
||||
assert.equal(base64ToBase64Url('+/8='), '-_8');
|
||||
});
|
||||
|
||||
it('requests a repository-scoped installation token', async () => {
|
||||
let request;
|
||||
const token = await createInstallationToken(CONFIG, {
|
||||
nowSeconds: 1_000,
|
||||
execute: (command, args) => {
|
||||
assert.equal(command, 'az');
|
||||
assert.ok(args.includes('RS256'));
|
||||
return '+/8=\n';
|
||||
},
|
||||
fetch: async (url, options) => {
|
||||
request = { url, options };
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ token: 'installation-token' }),
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(token, 'installation-token');
|
||||
assert.equal(request.url, 'https://api.github.com/app/installations/12345/access_tokens');
|
||||
assert.match(request.options.headers.Authorization, /^Bearer [^.]+\.[^.]+\.-_8$/);
|
||||
assert.deepEqual(JSON.parse(request.options.body), {
|
||||
repositories: ['agent-framework'],
|
||||
permissions: {
|
||||
contents: 'read',
|
||||
issues: 'write',
|
||||
members: 'read',
|
||||
pull_requests: 'write',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects incomplete configuration', () => {
|
||||
assert.throws(
|
||||
() => readConfig({}),
|
||||
/Required GitHub App authentication configuration is missing/,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects repository values with extra path segments before signing', async () => {
|
||||
let signed = false;
|
||||
|
||||
await assert.rejects(
|
||||
createInstallationToken(
|
||||
{ ...CONFIG, targetRepository: 'microsoft/agent-framework/extra' },
|
||||
{
|
||||
execute: () => {
|
||||
signed = true;
|
||||
return '+/8=\n';
|
||||
},
|
||||
},
|
||||
),
|
||||
/TARGET_REPOSITORY must use the owner\/repository format/,
|
||||
);
|
||||
assert.equal(signed, false);
|
||||
});
|
||||
|
||||
it('rejects an empty Key Vault signature', async () => {
|
||||
await assert.rejects(
|
||||
createInstallationToken(CONFIG, {
|
||||
execute: () => '\n',
|
||||
}),
|
||||
/Key Vault returned an empty signature/,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a failed GitHub token request', async () => {
|
||||
await assert.rejects(
|
||||
createInstallationToken(CONFIG, {
|
||||
execute: () => '+/8=\n',
|
||||
fetch: async () => ({ ok: false, status: 403 }),
|
||||
}),
|
||||
/GitHub installation token request failed with HTTP 403/,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects an empty GitHub installation token', async () => {
|
||||
await assert.rejects(
|
||||
createInstallationToken(CONFIG, {
|
||||
execute: () => '+/8=\n',
|
||||
fetch: async () => ({
|
||||
ok: true,
|
||||
json: async () => ({ token: '' }),
|
||||
}),
|
||||
}),
|
||||
/GitHub returned an empty installation token/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,341 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
/**
|
||||
* Tests for pr_limit_moderation.js.
|
||||
*
|
||||
* Run with: node --test .github/tests/test_pr_limit_moderation.js
|
||||
*/
|
||||
|
||||
const { describe, it } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const { enforcePrLimit } = require('../scripts/pr_limit_moderation.js');
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createContext({ author = 'community-user', authorType = 'User', labels = [], number = 123 } = {}) {
|
||||
return {
|
||||
repo: {
|
||||
owner: 'microsoft',
|
||||
repo: 'agent-framework',
|
||||
},
|
||||
payload: {
|
||||
pull_request: {
|
||||
number,
|
||||
labels: labels.map((name) => ({ name })),
|
||||
user: {
|
||||
login: author,
|
||||
type: authorType,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createCore() {
|
||||
const messages = [];
|
||||
return {
|
||||
messages,
|
||||
info(message) {
|
||||
messages.push(message);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createGithub({
|
||||
itemNumbers,
|
||||
labelExists = true,
|
||||
pullRequests = createPullRequestPage({ numbers: itemNumbers }),
|
||||
}) {
|
||||
const calls = [];
|
||||
|
||||
return {
|
||||
calls,
|
||||
async paginate(method, params) {
|
||||
calls.push({ api: 'paginate', method, params });
|
||||
return pullRequests;
|
||||
},
|
||||
rest: {
|
||||
issues: {
|
||||
async getLabel(params) {
|
||||
calls.push({ api: 'issues.getLabel', params });
|
||||
if (!labelExists) {
|
||||
const error = new Error('Not Found');
|
||||
error.status = 404;
|
||||
throw error;
|
||||
}
|
||||
return { data: { name: params.name } };
|
||||
},
|
||||
async createLabel(params) {
|
||||
calls.push({ api: 'issues.createLabel', params });
|
||||
return { data: { name: params.name } };
|
||||
},
|
||||
async addLabels(params) {
|
||||
calls.push({ api: 'issues.addLabels', params });
|
||||
return { data: [] };
|
||||
},
|
||||
async createComment(params) {
|
||||
calls.push({ api: 'issues.createComment', params });
|
||||
return { data: { id: 1 } };
|
||||
},
|
||||
},
|
||||
pulls: {
|
||||
async list(params) {
|
||||
calls.push({ api: 'pulls.list', params });
|
||||
return { data: pullRequests };
|
||||
},
|
||||
async update(params) {
|
||||
calls.push({ api: 'pulls.update', params });
|
||||
return { data: { state: params.state } };
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createPullRequestPage({ author = 'community-user', numbers }) {
|
||||
return numbers.map((number) => ({
|
||||
number,
|
||||
user: {
|
||||
login: author,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PR limit enforcement
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('PR limit enforcement', () => {
|
||||
it('does not close the PR when the author is at the open PR limit', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 123],
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext(),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.openPrCount, 10);
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
['paginate'],
|
||||
);
|
||||
});
|
||||
|
||||
it('counts the new PR when the pull list includes it', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext(),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.openPrCount, 11);
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
[
|
||||
'paginate',
|
||||
'issues.getLabel',
|
||||
'issues.addLabels',
|
||||
'issues.createComment',
|
||||
'pulls.update',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
it('counts the current PR on top of existing open PRs', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, ...Array.from({ length: 24 }, (_, index) => index + 1)],
|
||||
pullRequests: createPullRequestPage({
|
||||
numbers: [123, ...Array.from({ length: 25 }, (_, index) => index + 1)],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext(),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.openPrCount, 26);
|
||||
const comment = github.calls.find((call) => call.api === 'issues.createComment').params.body;
|
||||
assert.match(comment, /This PR would put you at 26 open pull requests/);
|
||||
});
|
||||
|
||||
it('creates the label when it does not already exist', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
labelExists: false,
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext(),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, true);
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
[
|
||||
'paginate',
|
||||
'issues.getLabel',
|
||||
'issues.createLabel',
|
||||
'issues.addLabels',
|
||||
'issues.createComment',
|
||||
'pulls.update',
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
github.calls.find((call) => call.api === 'issues.createLabel').params.name,
|
||||
'too-many-prs',
|
||||
);
|
||||
});
|
||||
|
||||
it('tolerates a 422 race when creating the label', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
labelExists: false,
|
||||
});
|
||||
github.rest.issues.createLabel = async (params) => {
|
||||
github.calls.push({ api: 'issues.createLabel', params });
|
||||
const error = new Error('Validation Failed');
|
||||
error.status = 422;
|
||||
throw error;
|
||||
};
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext(),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, true);
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
[
|
||||
'paginate',
|
||||
'issues.getLabel',
|
||||
'issues.createLabel',
|
||||
'issues.addLabels',
|
||||
'issues.createComment',
|
||||
'pulls.update',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
it('uses a diplomatic close message with the configured limit', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
pullRequests: createPullRequestPage({
|
||||
author: 'octo-contributor',
|
||||
numbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
}),
|
||||
});
|
||||
|
||||
await enforcePrLimit({
|
||||
github,
|
||||
context: createContext({ author: 'octo-contributor' }),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
const comment = github.calls.find((call) => call.api === 'issues.createComment').params.body;
|
||||
assert.match(comment, /Thank you for your contribution/);
|
||||
assert.match(comment, /limit community contributors to 10 open pull requests/);
|
||||
assert.match(comment, /@octo-contributor/);
|
||||
assert.match(comment, /`pr-limit-exempt` label and reopen/);
|
||||
});
|
||||
|
||||
it('does not close an exempt PR when it is reopened', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext({ labels: ['PR-LIMIT-EXEMPT'] }),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.exempt, true);
|
||||
assert.equal(result.openPrCount, null);
|
||||
assert.deepEqual(github.calls, []);
|
||||
});
|
||||
|
||||
it('does not close Dependabot PRs', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, ...Array.from({ length: 25 }, (_, index) => index + 1)],
|
||||
pullRequests: createPullRequestPage({
|
||||
author: 'dependabot[bot]',
|
||||
numbers: [123, ...Array.from({ length: 25 }, (_, index) => index + 1)],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext({ author: 'dependabot[bot]', authorType: 'Bot' }),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.dependabotExempt, true);
|
||||
assert.equal(result.openPrCount, null);
|
||||
assert.deepEqual(github.calls, []);
|
||||
});
|
||||
|
||||
it('counts the current PR when the author has more than one page of open PRs', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, ...Array.from({ length: 100 }, (_, index) => index + 1)],
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext({ number: 123 }),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.openPrCount, 101);
|
||||
});
|
||||
});
|
||||
@@ -1,134 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
# ruff:file-ignore[implicit-namespace-package, undocumented-public-class, undocumented-public-method]
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_PATH = Path(__file__).parents[1] / "scripts" / "python_check_coverage.py"
|
||||
SPEC = importlib.util.spec_from_file_location("python_check_coverage", SCRIPT_PATH)
|
||||
if SPEC is None or SPEC.loader is None:
|
||||
raise RuntimeError(f"Unable to load {SCRIPT_PATH}")
|
||||
coverage_checker = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = coverage_checker
|
||||
SPEC.loader.exec_module(coverage_checker)
|
||||
|
||||
|
||||
class CoveragePolicyTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temp_dir = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp_dir.name)
|
||||
self.packages_dir = self.root / "packages"
|
||||
self.packages_dir.mkdir()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def write_package(self, directory: str, name: str, status: str) -> None:
|
||||
package_dir = self.packages_dir / directory
|
||||
package_dir.mkdir()
|
||||
(package_dir / "pyproject.toml").write_text(
|
||||
f"""
|
||||
[project]
|
||||
name = "{name}"
|
||||
classifiers = ["Development Status :: {status}"]
|
||||
""".strip()
|
||||
)
|
||||
|
||||
def write_coverage(self, files: dict[str, list[int]]) -> Path:
|
||||
classes = []
|
||||
total_lines = 0
|
||||
covered_lines = 0
|
||||
for file_path, hits in files.items():
|
||||
lines = []
|
||||
for line_number, hit_count in enumerate(hits, start=1):
|
||||
total_lines += 1
|
||||
covered_lines += hit_count > 0
|
||||
lines.append(f'<line number="{line_number}" hits="{hit_count}"/>')
|
||||
classes.append(f'<class filename="{file_path}"><lines>{"".join(lines)}</lines></class>')
|
||||
|
||||
line_rate = covered_lines / total_lines if total_lines else 0
|
||||
xml_path = self.root / "coverage.xml"
|
||||
xml_path.write_text(
|
||||
f"""
|
||||
<coverage line-rate="{line_rate}" branch-rate="0">
|
||||
<packages>
|
||||
<package name="test">
|
||||
<classes>{"".join(classes)}</classes>
|
||||
</package>
|
||||
</packages>
|
||||
</coverage>
|
||||
""".strip()
|
||||
)
|
||||
return xml_path
|
||||
|
||||
def test_load_package_policies_uses_lifecycle_exemptions(self) -> None:
|
||||
self.write_package("alpha", "agent-framework-alpha", "3 - Alpha")
|
||||
self.write_package("beta", "agent-framework-beta", "4 - Beta")
|
||||
self.write_package("stable", "agent-framework-stable", "5 - Production/Stable")
|
||||
self.write_package("devui", "agent-framework-devui", "4 - Beta")
|
||||
self.write_package("lab", "agent-framework-lab", "4 - Beta")
|
||||
|
||||
policies = {policy.directory: policy for policy in coverage_checker.load_package_policies(self.packages_dir)}
|
||||
|
||||
self.assertFalse(policies["alpha"].enforced)
|
||||
self.assertTrue(policies["beta"].enforced)
|
||||
self.assertTrue(policies["stable"].enforced)
|
||||
self.assertTrue(policies["devui"].exempt)
|
||||
self.assertFalse(policies["devui"].enforced)
|
||||
self.assertTrue(policies["lab"].exempt)
|
||||
self.assertFalse(policies["lab"].enforced)
|
||||
|
||||
def test_load_package_policies_rejects_missing_lifecycle(self) -> None:
|
||||
package_dir = self.packages_dir / "missing"
|
||||
package_dir.mkdir()
|
||||
(package_dir / "pyproject.toml").write_text('[project]\nname = "agent-framework-missing"\n')
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "exactly one Development Status"):
|
||||
coverage_checker.load_package_policies(self.packages_dir)
|
||||
|
||||
def test_parse_coverage_aggregates_nested_modules_by_distribution(self) -> None:
|
||||
xml_path = self.write_coverage({
|
||||
"packages/core/agent_framework/_agents.py": [1, 0],
|
||||
"packages/core/agent_framework/_workflows/_workflow.py": [1, 1],
|
||||
})
|
||||
|
||||
package_stats, _, _ = coverage_checker.parse_coverage_xml(xml_path)
|
||||
|
||||
self.assertEqual(package_stats["core"].lines_valid, 4)
|
||||
self.assertEqual(package_stats["core"].lines_covered, 3)
|
||||
|
||||
def test_beta_package_below_threshold_fails(self) -> None:
|
||||
self.write_package("beta", "agent-framework-beta", "4 - Beta")
|
||||
xml_path = self.write_coverage({"packages/beta/agent_framework_beta/client.py": [1, 0]})
|
||||
|
||||
self.assertFalse(coverage_checker.check_coverage(xml_path, 85, self.packages_dir))
|
||||
|
||||
def test_missing_beta_package_fails(self) -> None:
|
||||
self.write_package("beta", "agent-framework-beta", "4 - Beta")
|
||||
xml_path = self.write_coverage({})
|
||||
|
||||
self.assertFalse(coverage_checker.check_coverage(xml_path, 85, self.packages_dir))
|
||||
|
||||
def test_alpha_and_exempt_packages_do_not_fail(self) -> None:
|
||||
self.write_package("alpha", "agent-framework-alpha", "3 - Alpha")
|
||||
self.write_package("devui", "agent-framework-devui", "4 - Beta")
|
||||
self.write_package("lab", "agent-framework-lab", "4 - Beta")
|
||||
xml_path = self.write_coverage({})
|
||||
|
||||
self.assertTrue(coverage_checker.check_coverage(xml_path, 85, self.packages_dir))
|
||||
|
||||
def test_beta_package_at_threshold_passes(self) -> None:
|
||||
self.write_package("beta", "agent-framework-beta", "4 - Beta")
|
||||
xml_path = self.write_coverage({"packages/beta/agent_framework_beta/client.py": [1] * 17 + [0] * 3})
|
||||
|
||||
self.assertTrue(coverage_checker.check_coverage(xml_path, 85, self.packages_dir))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,212 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
/**
|
||||
* Tests for resolve_integration_test_target.js.
|
||||
*
|
||||
* Run with: node --test .github/tests/test_resolve_integration_test_target.js
|
||||
*/
|
||||
|
||||
const { describe, it } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const resolveIntegrationTestTarget = require('../scripts/resolve_integration_test_target.js');
|
||||
|
||||
const HEAD_SHA = 'a'.repeat(40);
|
||||
const BASE_SHA = 'b'.repeat(40);
|
||||
|
||||
function review({
|
||||
id,
|
||||
login,
|
||||
state = 'APPROVED',
|
||||
commitId = HEAD_SHA,
|
||||
submittedAt = `2026-07-13T00:00:${String(id).padStart(2, '0')}Z`,
|
||||
}) {
|
||||
return {
|
||||
id,
|
||||
state,
|
||||
commit_id: commitId,
|
||||
submitted_at: submittedAt,
|
||||
user: { login },
|
||||
};
|
||||
}
|
||||
|
||||
function createMocks({
|
||||
pullState = 'open',
|
||||
pullAuthor = 'contributor',
|
||||
reviews = [],
|
||||
permissions = {},
|
||||
} = {}) {
|
||||
const core = {
|
||||
infoMessages: [],
|
||||
info(message) {
|
||||
this.infoMessages.push(message);
|
||||
},
|
||||
};
|
||||
const context = {
|
||||
repo: { owner: 'microsoft', repo: 'agent-framework' },
|
||||
};
|
||||
const github = {
|
||||
paginate: async () => reviews,
|
||||
rest: {
|
||||
pulls: {
|
||||
get: async () => ({
|
||||
data: {
|
||||
state: pullState,
|
||||
user: { login: pullAuthor },
|
||||
head: { sha: HEAD_SHA },
|
||||
base: { sha: BASE_SHA },
|
||||
},
|
||||
}),
|
||||
listReviews: async () => {},
|
||||
},
|
||||
repos: {
|
||||
get: async () => ({ data: { default_branch: 'main' } }),
|
||||
getBranch: async ({ branch }) => ({
|
||||
data: { commit: { sha: branch === 'main' ? BASE_SHA : HEAD_SHA } },
|
||||
}),
|
||||
getCollaboratorPermissionLevel: async ({ username }) => ({
|
||||
data: permissions[username] || {
|
||||
permission: 'read',
|
||||
user: { permissions: { push: false } },
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return { core, context, github };
|
||||
}
|
||||
|
||||
const WRITE_PERMISSION = {
|
||||
permission: 'write',
|
||||
user: { permissions: { push: true } },
|
||||
};
|
||||
|
||||
describe('input validation', () => {
|
||||
it('rejects missing and conflicting targets', async () => {
|
||||
const mocks = createMocks();
|
||||
await assert.rejects(
|
||||
() => resolveIntegrationTestTarget(mocks),
|
||||
/provide either a PR number or a branch name/,
|
||||
);
|
||||
await assert.rejects(
|
||||
() => resolveIntegrationTestTarget({ ...mocks, prNumber: '1', branch: 'feature' }),
|
||||
/not both/,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects invalid PR numbers and branch names', async () => {
|
||||
const mocks = createMocks();
|
||||
await assert.rejects(
|
||||
() => resolveIntegrationTestTarget({ ...mocks, prNumber: '1;echo' }),
|
||||
/Invalid PR number/,
|
||||
);
|
||||
await assert.rejects(
|
||||
() => resolveIntegrationTestTarget({ ...mocks, branch: 'feature branch' }),
|
||||
/Invalid branch name/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pull request resolution', () => {
|
||||
it('pins an open PR with two fresh write-capable approvals', async () => {
|
||||
const mocks = createMocks({
|
||||
reviews: [
|
||||
review({ id: 1, login: 'maintainer-one' }),
|
||||
review({ id: 2, login: 'maintainer-two' }),
|
||||
],
|
||||
permissions: {
|
||||
'maintainer-one': WRITE_PERMISSION,
|
||||
'maintainer-two': WRITE_PERMISSION,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await resolveIntegrationTestTarget({ ...mocks, prNumber: '123' });
|
||||
|
||||
assert.deepEqual(result, {
|
||||
baseRef: BASE_SHA,
|
||||
checkoutRef: HEAD_SHA,
|
||||
description: 'PR #123',
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects closed PRs', async () => {
|
||||
const mocks = createMocks({ pullState: 'closed' });
|
||||
await assert.rejects(
|
||||
() => resolveIntegrationTestTarget({ ...mocks, prNumber: '123' }),
|
||||
/is not open/,
|
||||
);
|
||||
});
|
||||
|
||||
it('ignores stale, self, and read-only approvals', async () => {
|
||||
const mocks = createMocks({
|
||||
reviews: [
|
||||
review({ id: 1, login: 'stale', commitId: 'c'.repeat(40) }),
|
||||
review({ id: 2, login: 'contributor' }),
|
||||
review({ id: 3, login: 'reader' }),
|
||||
review({ id: 4, login: 'maintainer' }),
|
||||
],
|
||||
permissions: {
|
||||
contributor: WRITE_PERMISSION,
|
||||
reader: { permission: 'read', user: { permissions: { push: false } } },
|
||||
maintainer: WRITE_PERMISSION,
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() => resolveIntegrationTestTarget({ ...mocks, prNumber: '123' }),
|
||||
/found 1/,
|
||||
);
|
||||
});
|
||||
|
||||
it('uses each reviewer latest decisive review and ignores later comments', async () => {
|
||||
const mocks = createMocks({
|
||||
reviews: [
|
||||
review({ id: 1, login: 'changes-requested' }),
|
||||
review({ id: 2, login: 'changes-requested', state: 'CHANGES_REQUESTED' }),
|
||||
review({ id: 3, login: 'maintainer-one' }),
|
||||
review({ id: 4, login: 'maintainer-one', state: 'COMMENTED' }),
|
||||
review({ id: 5, login: 'maintainer-two' }),
|
||||
],
|
||||
permissions: {
|
||||
'changes-requested': WRITE_PERMISSION,
|
||||
'maintainer-one': WRITE_PERMISSION,
|
||||
'maintainer-two': WRITE_PERMISSION,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await resolveIntegrationTestTarget({ ...mocks, prNumber: '123' });
|
||||
assert.equal(result.checkoutRef, HEAD_SHA);
|
||||
});
|
||||
|
||||
it('does not count a dismissed approval', async () => {
|
||||
const mocks = createMocks({
|
||||
reviews: [
|
||||
review({ id: 1, login: 'dismissed', state: 'DISMISSED' }),
|
||||
review({ id: 2, login: 'maintainer' }),
|
||||
],
|
||||
permissions: {
|
||||
dismissed: WRITE_PERMISSION,
|
||||
maintainer: WRITE_PERMISSION,
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() => resolveIntegrationTestTarget({ ...mocks, prNumber: '123' }),
|
||||
/found 1/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('branch resolution', () => {
|
||||
it('pins base-repository branches and their comparison base to SHAs', async () => {
|
||||
const mocks = createMocks();
|
||||
const result = await resolveIntegrationTestTarget({ ...mocks, branch: 'feature/test' });
|
||||
|
||||
assert.deepEqual(result, {
|
||||
baseRef: BASE_SHA,
|
||||
checkoutRef: HEAD_SHA,
|
||||
description: 'branch feature/test',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -23,16 +23,16 @@ For each project that needs to be migrated, you need to do the following:
|
||||
- Identify the specific Semantic Kernel agent types being used:
|
||||
- `ChatCompletionAgent` → `ChatClientAgent`
|
||||
- `OpenAIAssistantAgent` → `assistantsClient.CreateAIAgent()` (via OpenAI Assistants client extension)
|
||||
- `AzureAIAgent` → `persistentAgentsClient.CreateAIAgent()` (via Microsoft Foundry client extension)
|
||||
- `AzureAIAgent` → `persistentAgentsClient.CreateAIAgent()` (via Azure AI Foundry client extension)
|
||||
- `OpenAIResponseAgent` → `responsesClient.CreateAIAgent()` (via OpenAI Responses client extension)
|
||||
- `A2AAgent` → `AIAgent` (via A2A card resolver)
|
||||
- `BedrockAgent` → Custom implementation required (not supported)
|
||||
- Determine if agents are being created new or retrieved from hosted services:
|
||||
- **New agents**: Use `CreateAIAgent()` methods
|
||||
- **Existing hosted agents**: Use `GetAIAgent(agentId)` methods for OpenAI Assistants and Microsoft Foundry
|
||||
- **Existing hosted agents**: Use `GetAIAgent(agentId)` methods for OpenAI Assistants and Azure AI Foundry
|
||||
</agent_type_identification>
|
||||
|
||||
- Determine the AI provider being used (OpenAI, Azure OpenAI, Microsoft Foundry, etc.)
|
||||
- Determine the AI provider being used (OpenAI, Azure OpenAI, Azure AI Foundry, etc.)
|
||||
- Analyze tool/function registration patterns
|
||||
- Review thread management and invocation patterns
|
||||
|
||||
@@ -90,7 +90,7 @@ below in wrong order or skip any of them):
|
||||
you generate report when migration complete. Report should contain:
|
||||
- all project dependencies changes (mention what was changed, added or removed, including provider-specific packages)
|
||||
- all code files that were changed (mention what was changed in the file, if it was not changed, just mention that the file was not changed)
|
||||
- provider-specific migration patterns used (OpenAI, Azure OpenAI, Microsoft Foundry, A2A, ONNX, etc.)
|
||||
- provider-specific migration patterns used (OpenAI, Azure OpenAI, Azure AI Foundry, A2A, ONNX, etc.)
|
||||
- all cases where you could not convert the code because of unsupported features and you were unable to find a workaround
|
||||
- unsupported providers that require custom implementation (Bedrock, CopilotStudio)
|
||||
- breaking glass pattern migrations (InnerContent → RawRepresentation) and any CodeInterpreter or advanced tool usage
|
||||
@@ -223,7 +223,7 @@ using Microsoft.Agents.AI;
|
||||
// Provider-specific namespaces (add only if needed):
|
||||
using OpenAI; // For OpenAI provider
|
||||
using Azure.AI.OpenAI; // For Azure OpenAI provider
|
||||
using Azure.AI.Agents.Persistent; // For Microsoft Foundry provider
|
||||
using Azure.AI.Agents.Persistent; // For Azure AI Foundry provider
|
||||
using Azure.Identity; // For Azure authentication
|
||||
```
|
||||
</configuration_changes>
|
||||
@@ -499,7 +499,7 @@ For every thread created if there's intent to cleanup, the caller should track a
|
||||
var assistantClient = new OpenAIClient(apiKey).GetAssistantClient();
|
||||
await assistantClient.DeleteThreadAsync(thread.ConversationId);
|
||||
|
||||
// For Microsoft Foundry (when cleanup is needed):
|
||||
// For Azure AI Foundry (when cleanup is needed):
|
||||
var persistentClient = new PersistentAgentsClient(endpoint, credential);
|
||||
await persistentClient.Threads.DeleteThreadAsync(thread.ConversationId);
|
||||
|
||||
@@ -514,7 +514,7 @@ await persistentClient.Threads.DeleteThreadAsync(thread.ConversationId);
|
||||
1. Remove `thread.DeleteAsync()` calls
|
||||
2. Use provider-specific client for cleanup when required
|
||||
3. Access thread ID via `thread.ConversationId` property
|
||||
4. Only implement cleanup for providers that require it (Assistants, Microsoft Foundry)
|
||||
4. Only implement cleanup for providers that require it (Assistants, Azure AI Foundry)
|
||||
</api_changes>
|
||||
|
||||
### Provider-Specific Creation Patterns
|
||||
@@ -550,13 +550,13 @@ AIAgent agent = new AzureOpenAIClient(endpoint, credential)
|
||||
.CreateAIAgent(instructions: instructions);
|
||||
```
|
||||
|
||||
**Microsoft Foundry (New):**
|
||||
**Azure AI Foundry (New):**
|
||||
```csharp
|
||||
AIAgent agent = new PersistentAgentsClient(endpoint, credential)
|
||||
.CreateAIAgent(model: deploymentName, instructions: instructions);
|
||||
```
|
||||
|
||||
**Microsoft Foundry (Existing):**
|
||||
**Azure AI Foundry (Existing):**
|
||||
```csharp
|
||||
AIAgent agent = await new PersistentAgentsClient(endpoint, credential)
|
||||
.GetAIAgentAsync(agentId);
|
||||
@@ -1079,7 +1079,7 @@ AgentThread thread = agent.GetNewThread();
|
||||
```
|
||||
</api_changes>
|
||||
|
||||
### 4. Microsoft Foundry (AzureAIAgent) Migration
|
||||
### 4. Azure AI Foundry (AzureAIAgent) Migration
|
||||
|
||||
<configuration_changes>
|
||||
**Remove Semantic Kernel Packages:**
|
||||
|
||||
@@ -32,13 +32,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -64,6 +64,6 @@ jobs:
|
||||
# ./location_of_script_within_repo/buildscript.sh
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
|
||||
uses: github/codeql-action/analyze@v4
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
name: DevFlow PR Review
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types:
|
||||
- opened
|
||||
- reopened
|
||||
- ready_for_review
|
||||
issue_comment:
|
||||
types:
|
||||
- created
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: Pull request number to review
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: devflow-pr-review-${{ github.repository }}-${{ github.event.pull_request.number || github.event.issue.number || inputs.pr_number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
DEVFLOW_REPOSITORY: ${{ vars.DF_REPO }}
|
||||
DEVFLOW_REF: main
|
||||
TARGET_REPO_PATH: ${{ github.workspace }}/target-repo
|
||||
DEVFLOW_PATH: ${{ github.workspace }}/devflow
|
||||
MODEL_CONFIG_PATH: ${{ github.workspace }}/devflow/config.ci.yaml
|
||||
|
||||
jobs:
|
||||
team_check:
|
||||
if: >-
|
||||
github.event_name != 'issue_comment' ||
|
||||
(
|
||||
github.event.issue.pull_request &&
|
||||
github.event.comment.body == '/review' &&
|
||||
(
|
||||
github.event.comment.author_association == 'MEMBER' ||
|
||||
github.event.comment.author_association == 'OWNER'
|
||||
)
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
environment: github-app-auth
|
||||
outputs:
|
||||
is_team_member: ${{ steps.check.outputs.is_team_member }}
|
||||
pr_number: ${{ steps.pr.outputs.pr_number }}
|
||||
pr_url: ${{ steps.pr.outputs.pr_url }}
|
||||
repo: ${{ steps.pr.outputs.repo }}
|
||||
steps:
|
||||
- name: Resolve PR metadata
|
||||
id: pr
|
||||
shell: bash
|
||||
env:
|
||||
PR_HTML_URL: ${{ github.event.pull_request.html_url }}
|
||||
PR_NUMBER_COMMENT: ${{ github.event.issue.number }}
|
||||
PR_NUMBER_EVENT: ${{ github.event.pull_request.number }}
|
||||
PR_NUMBER_INPUT: ${{ inputs.pr_number }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then
|
||||
pr_number="${PR_NUMBER_EVENT}"
|
||||
pr_url="${PR_HTML_URL}"
|
||||
elif [[ "${GITHUB_EVENT_NAME}" == "issue_comment" ]]; then
|
||||
pr_number="${PR_NUMBER_COMMENT}"
|
||||
pr_url="https://github.com/${GITHUB_REPOSITORY}/pull/${pr_number}"
|
||||
else
|
||||
pr_number="${PR_NUMBER_INPUT}"
|
||||
pr_url="https://github.com/${GITHUB_REPOSITORY}/pull/${pr_number}"
|
||||
fi
|
||||
|
||||
if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "Could not determine PR number; for workflow_dispatch runs, the 'pr_number' input is required when not running on pull_request_target." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "pr_url=${pr_url}" >> "$GITHUB_OUTPUT"
|
||||
echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT"
|
||||
echo "repo=${GITHUB_REPOSITORY}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout GitHub automation
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.sha }}
|
||||
sparse-checkout: |
|
||||
.github/actions/github-app-token
|
||||
.github/scripts/check_team_membership.js
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get GitHub automation token
|
||||
id: github-auth
|
||||
uses: ./.github/actions/github-app-token
|
||||
with:
|
||||
mode: ${{ vars.GH_APP_AUTH_MODE }}
|
||||
azure-client-id: ${{ secrets.GH_APP_AZURE_CLIENT_ID }}
|
||||
azure-tenant-id: ${{ secrets.GH_APP_AZURE_TENANT_ID }}
|
||||
azure-subscription-id: ${{ secrets.GH_APP_AZURE_SUBSCRIPTION_ID }}
|
||||
key-vault-name: ${{ secrets.GH_APP_KEY_VAULT_NAME }}
|
||||
key-name: ${{ secrets.GH_APP_KEY_NAME }}
|
||||
github-app-client-id: ${{ secrets.GH_APP_CLIENT_ID }}
|
||||
github-app-installation-id: ${{ secrets.GH_APP_INSTALLATION_ID }}
|
||||
repository: ${{ github.repository }}
|
||||
fallback-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
|
||||
- name: Check review requester team membership
|
||||
id: check
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
env:
|
||||
MEMBERSHIP_USER: ${{ github.event_name == 'issue_comment' && github.event.comment.user.login || '' }}
|
||||
TEAM_NAME: ${{ secrets.DEVELOPER_TEAM }}
|
||||
PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
|
||||
with:
|
||||
github-token: ${{ steps.github-auth.outputs.token }}
|
||||
script: |
|
||||
const checkTeamMembership = require('./.github/scripts/check_team_membership.js');
|
||||
const { author, isTeamMember } = await checkTeamMembership({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
teamSlug: process.env.TEAM_NAME,
|
||||
issueNumber: process.env.PR_NUMBER,
|
||||
username: process.env.MEMBERSHIP_USER,
|
||||
});
|
||||
core.setOutput('is_team_member', isTeamMember ? 'true' : 'false');
|
||||
if (isTeamMember) {
|
||||
core.info(`User ${author} is a team member; proceeding with review.`);
|
||||
} else {
|
||||
core.info(`User ${author} is not a member of ${process.env.TEAM_NAME}; skipping review.`);
|
||||
}
|
||||
|
||||
- name: React to authorized review command
|
||||
if: ${{ github.event_name == 'issue_comment' && steps.check.outputs.is_team_member == 'true' }}
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ steps.github-auth.outputs.token }}
|
||||
script: |
|
||||
await github.rest.reactions.createForIssueComment({
|
||||
...context.repo,
|
||||
comment_id: context.payload.comment.id,
|
||||
content: 'eyes',
|
||||
});
|
||||
|
||||
review:
|
||||
runs-on: ubuntu-latest
|
||||
needs: team_check
|
||||
if: ${{ needs.team_check.outputs.is_team_member == 'true' }}
|
||||
permissions:
|
||||
copilot-requests: write
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
timeout-minutes: 60
|
||||
# Advisory check: failures here should not block the PR. The reviewer
|
||||
# posts comments as a best-effort signal; if the pipeline breaks, the
|
||||
# PR author should still be able to merge without a red required check.
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
# Safe checkout: base repo only, not the untrusted PR head.
|
||||
- name: Checkout target repo base
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
path: target-repo
|
||||
|
||||
# Private DevFlow checkout: the PAT/token grants access to this repo's code.
|
||||
- name: Checkout DevFlow
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
repository: ${{ env.DEVFLOW_REPOSITORY }}
|
||||
ref: ${{ env.DEVFLOW_REF }}
|
||||
token: ${{ secrets.DEVFLOW_TOKEN }}
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
path: devflow
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
version: "0.11.x"
|
||||
enable-cache: true
|
||||
|
||||
- name: Install DevFlow dependencies
|
||||
working-directory: ${{ env.DEVFLOW_PATH }}
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: Run PR review
|
||||
id: review
|
||||
working-directory: ${{ env.DEVFLOW_PATH }}
|
||||
env:
|
||||
DEVFLOW_TOKEN: ${{ secrets.DEVFLOW_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
SK_REPO_PATH: ${{ env.TARGET_REPO_PATH }}
|
||||
AGENT_REPO_PATH: ${{ env.TARGET_REPO_PATH }}
|
||||
PR_URL: ${{ needs.team_check.outputs.pr_url }}
|
||||
run: |
|
||||
uv run python scripts/trigger_pr_review.py \
|
||||
--pr-url "$PR_URL" \
|
||||
--github-username "$GITHUB_ACTOR" \
|
||||
--review-compare \
|
||||
--no-require-comment-selection
|
||||
@@ -37,11 +37,9 @@ jobs:
|
||||
outputs:
|
||||
dotnetChanges: ${{ steps.filter.outputs.dotnet }}
|
||||
cosmosDbChanges: ${{ steps.filter.outputs.cosmosdb }}
|
||||
foundryHostingChanges: ${{ steps.filter.outputs.foundryHosting }}
|
||||
coreChanged: ${{ steps.filter.outputs.core }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
@@ -49,33 +47,6 @@ jobs:
|
||||
- 'dotnet/**'
|
||||
cosmosdb:
|
||||
- 'dotnet/src/Microsoft.Agents.AI.CosmosNoSql/**'
|
||||
# The Foundry hosted-agent IT is costly (builds a container, pushes to ACR,
|
||||
# provisions live agents). Only run it when the project under test, its
|
||||
# dependency chain, the test container, the test fixture, or their tooling
|
||||
# changed. Keep this list in sync with $hashedDirs in scripts/it-build-image.ps1.
|
||||
foundryHosting:
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Foundry/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Abstractions/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Workflows/**'
|
||||
- 'dotnet/tests/Foundry.Hosting.IntegrationTests/**'
|
||||
- 'dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/**'
|
||||
- 'dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-AzureSearchRag/**'
|
||||
- 'dotnet/Directory.Packages.props'
|
||||
- 'dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1'
|
||||
- '.github/workflows/dotnet-build-and-test.yml'
|
||||
core:
|
||||
- 'dotnet/src/Microsoft.Agents.AI/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Abstractions/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.OpenAI/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Workflows/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Workflows.Generators/**'
|
||||
- 'dotnet/eng/scripts/New-FilteredSolution.ps1'
|
||||
- 'dotnet/tests/Directory.Build.props'
|
||||
- 'dotnet/Directory.Packages.props'
|
||||
- 'dotnet/global.json'
|
||||
- '.github/workflows/dotnet-build-and-test.yml'
|
||||
# run only if 'dotnet' files were changed
|
||||
- name: dotnet tests
|
||||
if: steps.filter.outputs.dotnet == 'true'
|
||||
@@ -103,7 +74,7 @@ jobs:
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: |
|
||||
@@ -111,13 +82,10 @@ jobs:
|
||||
.github
|
||||
dotnet
|
||||
python
|
||||
declarative-agents
|
||||
|
||||
- name: Free runner disk space
|
||||
uses: ./.github/actions/free-runner-disk-space
|
||||
workflow-samples
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
|
||||
uses: actions/setup-dotnet@v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
- name: Build dotnet solutions
|
||||
@@ -155,7 +123,6 @@ jobs:
|
||||
# Change to project directory to ensure local nuget.config is used
|
||||
pushd consoleapp
|
||||
dotnet add packcheck.csproj package Microsoft.Agents.AI --prerelease
|
||||
dotnet add packcheck.csproj package Microsoft.Agents.AI.LocalCodeAct --prerelease
|
||||
dotnet build -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} packcheck.csproj
|
||||
|
||||
# Clean up
|
||||
@@ -177,7 +144,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
environment: ${{ matrix.environment }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: |
|
||||
@@ -185,10 +152,7 @@ jobs:
|
||||
.github
|
||||
dotnet
|
||||
python
|
||||
declarative-agents
|
||||
|
||||
- name: Free runner disk space
|
||||
uses: ./.github/actions/free-runner-disk-space
|
||||
workflow-samples
|
||||
|
||||
# Start Cosmos DB Emulator for all integration tests and only for unit tests when CosmosDB changes happened)
|
||||
- name: Start Azure Cosmos DB Emulator
|
||||
@@ -201,7 +165,7 @@ jobs:
|
||||
echo "COSMOSDB_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
|
||||
uses: actions/setup-dotnet@v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
|
||||
@@ -230,10 +194,10 @@ jobs:
|
||||
Verbose = $true
|
||||
}
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
|
||||
-TestProjectNameIncludeFilter "*UnitTests*" `
|
||||
-TestProjectNameFilter "*UnitTests*" `
|
||||
-OutputPath dotnet/filtered-unit.slnx
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
|
||||
-TestProjectNameIncludeFilter "*IntegrationTests*" `
|
||||
-TestProjectNameFilter "*IntegrationTests*" `
|
||||
-OutputPath dotnet/filtered-integration.slnx
|
||||
|
||||
- name: Run Unit Tests
|
||||
@@ -269,12 +233,20 @@ jobs:
|
||||
|
||||
- name: Azure CLI Login
|
||||
if: github.event_name != 'pull_request' && matrix.integration-tests
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# This setup action is required for both Durable Task and Azure Functions integration tests.
|
||||
# We only run it on Ubuntu since the Durable Task and Azure Functions features are not available
|
||||
# on .NET Framework (net472) which is what we use the Windows runner for.
|
||||
- name: Set up Durable Task and Azure Functions Integration Test Emulators
|
||||
if: github.event_name != 'pull_request' && matrix.integration-tests && matrix.os == 'ubuntu-latest'
|
||||
uses: ./.github/actions/azure-functions-integration-setup
|
||||
id: azure-functions-setup
|
||||
|
||||
- name: Run Integration Tests
|
||||
shell: pwsh
|
||||
working-directory: dotnet
|
||||
@@ -285,11 +257,8 @@ jobs:
|
||||
-c ${{ matrix.configuration }} `
|
||||
--no-build -v Normal `
|
||||
--report-xunit-trx `
|
||||
--report-junit `
|
||||
--results-directory ../IntegrationTestResults/ `
|
||||
--ignore-exit-code 8 `
|
||||
--filter-not-trait "Category=IntegrationDisabled" `
|
||||
--filter-not-trait "Category=FoundryHostedAgents" `
|
||||
--parallel-algorithm aggressive `
|
||||
--max-threads 2.0x
|
||||
env:
|
||||
@@ -304,20 +273,15 @@ jobs:
|
||||
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
|
||||
# Microsoft Foundry
|
||||
# Azure AI Foundry
|
||||
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
|
||||
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
|
||||
# Anthropic Models
|
||||
# Disable Anthropic tests by not providing environment vars until 404 failure is resolved
|
||||
# ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
# ANTHROPIC_CHAT_MODEL_NAME: ${{ vars.ANTHROPIC_CHAT_MODEL_NAME }}
|
||||
# ANTHROPIC_REASONING_MODEL_NAME: ${{ vars.ANTHROPIC_REASONING_MODEL_NAME }}
|
||||
|
||||
# Generate test reports and check coverage
|
||||
- name: Generate test reports
|
||||
if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
|
||||
uses: danielpalme/ReportGenerator-GitHub-Action@2a82782178b2816d9d6960a7345fdd164791b323 # 5.5.3
|
||||
uses: danielpalme/ReportGenerator-GitHub-Action@5.5.3
|
||||
with:
|
||||
reports: "./TestResults/Coverage/**/*.cobertura.xml"
|
||||
targetdir: "./TestResults/Reports"
|
||||
@@ -325,7 +289,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage report artifact
|
||||
if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: CoverageReport-${{ matrix.os }}-${{ matrix.targetFramework }}-${{ matrix.configuration }} # Artifact name
|
||||
path: ./TestResults/Reports # Directory containing files to upload
|
||||
@@ -335,107 +299,11 @@ jobs:
|
||||
shell: pwsh
|
||||
run: ./dotnet/eng/scripts/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD
|
||||
|
||||
- name: Upload integration test results
|
||||
if: always() && github.event_name != 'pull_request' && matrix.integration-tests
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: dotnet-test-results-${{ matrix.targetFramework }}-${{ matrix.os }}
|
||||
path: IntegrationTestResults/**/*.junit
|
||||
if-no-files-found: ignore
|
||||
|
||||
# The Foundry hosted-agent IT is costly (it builds a container, pushes to ACR, and provisions
|
||||
# live agents on a separate Foundry project). Running it in its own job keeps the overall
|
||||
# workflow time roughly flat: it executes in parallel to dotnet-build and dotnet-test and is
|
||||
# gated on paths-filter.outputs.foundryHostingChanges so unrelated edits skip the work.
|
||||
dotnet-foundry-hosted-it:
|
||||
needs: paths-filter
|
||||
if: github.event_name != 'pull_request' && needs.paths-filter.outputs.foundryHostingChanges == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
env:
|
||||
configuration: Release
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: |
|
||||
.
|
||||
.github
|
||||
dotnet
|
||||
python
|
||||
|
||||
- name: Free runner disk space
|
||||
uses: ./.github/actions/free-runner-disk-space
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
|
||||
# Build the test csproj directly instead of a filtered slnx + -f override.
|
||||
# The test project pins TargetFrameworks=net10.0 and its ProjectReference closure
|
||||
# gives MSBuild a single-rooted graph, so each multi-targeted dependency is invoked
|
||||
# exactly once for net10.0. This avoids the MSB3026/MSB3491/MSB4018/MSB3883 file-lock
|
||||
# collisions caused by parallel inner-builds racing on shared bin/obj output paths
|
||||
# under the previous slnx + global TFM override approach.
|
||||
- name: Build Foundry hosted IT (and its deps)
|
||||
shell: bash
|
||||
run: dotnet build dotnet/tests/Foundry.Hosting.IntegrationTests/Foundry.Hosting.IntegrationTests.csproj -c "$configuration" --warnaserror
|
||||
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# We rebuild and push the test container image on every IT run so framework code changes
|
||||
# are picked up; the image tag is content-hashed across the test container source AND its
|
||||
# framework project references, so identical content is a no-op push.
|
||||
#
|
||||
# The script always passes --no-dependencies to dotnet publish so publish never re-touches
|
||||
# the framework lib DLLs the prior "Build Foundry hosted IT (and its deps)" step produced.
|
||||
# This structurally eliminates the MSB3026 collision that VBCSCompiler from the prebuild
|
||||
# would otherwise cause by holding file handles to those DLLs. Do not remove the prebuild
|
||||
# step: the subsequent `dotnet test --no-build` step and the publish's ProjectReference
|
||||
# resolution both depend on the prebuilt outputs being present.
|
||||
- name: Build and push Foundry Hosted Agents test container
|
||||
id: build-foundry-hosted-image
|
||||
shell: pwsh
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
$registry = "${{ vars.IT_HOSTED_AGENT_REGISTRY }}"
|
||||
if ([string]::IsNullOrWhiteSpace($registry)) {
|
||||
throw "IT_HOSTED_AGENT_REGISTRY not set in the integration environment."
|
||||
}
|
||||
& "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry | Tee-Object -FilePath $env:GITHUB_ENV -Append
|
||||
|
||||
- name: Run Foundry Hosted Agents Integration Tests
|
||||
shell: pwsh
|
||||
working-directory: dotnet
|
||||
run: |
|
||||
dotnet test --project tests/Foundry.Hosting.IntegrationTests/Foundry.Hosting.IntegrationTests.csproj `
|
||||
-c $env:configuration `
|
||||
--no-build -v Normal `
|
||||
--report-xunit-trx `
|
||||
--ignore-exit-code 8 `
|
||||
--filter-trait "Category=FoundryHostedAgents"
|
||||
env:
|
||||
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.IT_HOSTED_AGENT_PROJECT_ENDPOINT }}
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.IT_HOSTED_AGENT_MODEL_DEPLOYMENT_NAME }}
|
||||
# Azure AI Search (for the azure-search-rag scenario). Reuses the integration
|
||||
# environment secrets shared with python-sample-validation.yml. The index is
|
||||
# provisioned out of band; see dotnet/tests/Foundry.Hosting.IntegrationTests/README.md
|
||||
# for the required schema and seed content.
|
||||
AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
|
||||
AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }}
|
||||
# IT_HOSTED_AGENT_IMAGE was exported into $GITHUB_ENV by the previous step.
|
||||
|
||||
# This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
|
||||
dotnet-build-and-test-check:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs: [dotnet-build, dotnet-test, dotnet-foundry-hosted-it]
|
||||
needs: [dotnet-build, dotnet-test]
|
||||
steps:
|
||||
- name: Get Date
|
||||
shell: bash
|
||||
@@ -463,74 +331,13 @@ jobs:
|
||||
- name: Fail workflow if tests failed
|
||||
id: check_tests_failed
|
||||
if: contains(join(needs.*.result, ','), 'failure')
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: core.setFailed('Integration Tests Failed!')
|
||||
|
||||
- name: Fail workflow if tests cancelled
|
||||
id: check_tests_cancelled
|
||||
if: contains(join(needs.*.result, ','), 'cancelled')
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: core.setFailed('Integration Tests Cancelled!')
|
||||
|
||||
# Integration test trend report (aggregates JUnit XML results from dotnet test jobs)
|
||||
dotnet-integration-test-report:
|
||||
name: Integration Test Report
|
||||
if: >
|
||||
always() &&
|
||||
github.event_name != 'pull_request' &&
|
||||
(contains(join(needs.*.result, ','), 'success') ||
|
||||
contains(join(needs.*.result, ','), 'failure'))
|
||||
needs: [dotnet-test]
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: |
|
||||
.github/actions/python-setup
|
||||
python
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: "3.13"
|
||||
os: ${{ runner.os }}
|
||||
- name: Download all test results from current run
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: dotnet-test-results-*
|
||||
path: dotnet-test-results/
|
||||
- name: Restore report history cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: python/dotnet-integration-report-history.json
|
||||
key: dotnet-integration-report-history-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
dotnet-integration-report-history-
|
||||
- name: Generate trend report
|
||||
run: >
|
||||
uv run python scripts/integration_test_report/aggregate.py
|
||||
../dotnet-test-results/
|
||||
dotnet-integration-report-history.json
|
||||
dotnet-integration-test-report.md
|
||||
- name: Post to Job Summary
|
||||
if: always()
|
||||
run: cat dotnet-integration-test-report.md >> $GITHUB_STEP_SUMMARY
|
||||
- name: Save report history cache
|
||||
if: always()
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: python/dotnet-integration-report-history.json
|
||||
key: dotnet-integration-report-history-${{ github.run_id }}
|
||||
- name: Upload trend report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: dotnet-integration-test-report
|
||||
path: |
|
||||
python/dotnet-integration-test-report.md
|
||||
python/dotnet-integration-report-history.json
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Get changed files
|
||||
id: changed-files
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: jitterbit/get-changed-files@b17fbb00bdc0c0f63fcf166580804b4d2cdc2a42 # v1
|
||||
uses: jitterbit/get-changed-files@v1
|
||||
continue-on-error: true
|
||||
|
||||
- name: No C# files changed
|
||||
@@ -54,14 +54,11 @@ jobs:
|
||||
- name: Find csproj files
|
||||
id: find-csproj
|
||||
if: github.event_name != 'pull_request' || steps.changed-files.outputs.added_modified != '' || steps.changed-files.outcome == 'failure'
|
||||
env:
|
||||
ADDED_MODIFIED: ${{ steps.changed-files.outputs.added_modified }}
|
||||
run: |
|
||||
csproj_files=()
|
||||
exclude_files=("Experimental.Orchestration.Flow.csproj" "Experimental.Orchestration.Flow.UnitTests.csproj" "Experimental.Orchestration.Flow.IntegrationTests.csproj")
|
||||
set -f
|
||||
if [[ ${{ steps.changed-files.outcome }} == 'success' ]]; then
|
||||
for file in $ADDED_MODIFIED; do
|
||||
for file in ${{ steps.changed-files.outputs.added_modified }}; do
|
||||
echo "$file was changed"
|
||||
dir="./$file"
|
||||
while [[ $dir != "." && $dir != "/" && $dir != $GITHUB_WORKSPACE ]]; do
|
||||
@@ -83,7 +80,6 @@ jobs:
|
||||
csproj_files=($(printf "%s\n" "${csproj_files[@]}" | sort -u))
|
||||
echo "Found ${#csproj_files[@]} unique csproj/slnx files: ${csproj_files[*]}"
|
||||
echo "csproj_files=${csproj_files[*]}" >> $GITHUB_OUTPUT
|
||||
set +f
|
||||
|
||||
- name: Pull container dotnet/sdk:${{ matrix.dotnet }}
|
||||
if: steps.find-csproj.outputs.csproj_files != ''
|
||||
@@ -92,11 +88,8 @@ jobs:
|
||||
# This step will run dotnet format on each of the unique csproj files and fail if any changes are made
|
||||
- name: Run dotnet format
|
||||
if: steps.find-csproj.outputs.csproj_files != ''
|
||||
env:
|
||||
CSPROJ_FILES: ${{ steps.find-csproj.outputs.csproj_files }}
|
||||
run: |
|
||||
set -f
|
||||
for csproj in $CSPROJ_FILES; do
|
||||
for csproj in ${{ steps.find-csproj.outputs.csproj_files }}; do
|
||||
echo "Running dotnet format on $csproj"
|
||||
docker run --rm -v "$(pwd):/app" -w /app mcr.microsoft.com/dotnet/sdk:${{ matrix.dotnet }} dotnet format "$csproj" --verify-no-changes --verbosity diagnostic
|
||||
docker run --rm -v $(pwd):/app -w /app mcr.microsoft.com/dotnet/sdk:${{ matrix.dotnet }} /bin/sh -c "dotnet format $csproj --verify-no-changes --verbosity diagnostic"
|
||||
done
|
||||
|
||||
@@ -9,30 +9,16 @@ on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
checkout-ref:
|
||||
description: "Immutable commit SHA to check out"
|
||||
description: "Git ref to checkout (e.g., refs/pull/123/head)"
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
AZURE_CLIENT_ID:
|
||||
required: true
|
||||
AZURE_TENANT_ID:
|
||||
required: true
|
||||
AZURE_SUBSCRIPTION_ID:
|
||||
required: true
|
||||
AZUREAI__ENDPOINT:
|
||||
required: true
|
||||
OPENAI__APIKEY:
|
||||
required: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
dotnet-integration-tests:
|
||||
permissions:
|
||||
copilot-requests: write
|
||||
contents: read
|
||||
id-token: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -43,7 +29,7 @@ jobs:
|
||||
environment: integration
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
@@ -52,7 +38,7 @@ jobs:
|
||||
.github
|
||||
dotnet
|
||||
python
|
||||
declarative-agents
|
||||
workflow-samples
|
||||
|
||||
- name: Start Azure Cosmos DB Emulator
|
||||
if: runner.os == 'Windows'
|
||||
@@ -64,7 +50,7 @@ jobs:
|
||||
echo "COSMOS_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
|
||||
uses: actions/setup-dotnet@v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
|
||||
@@ -77,12 +63,16 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Set up Durable Task and Azure Functions Integration Test Emulators
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
uses: ./.github/actions/azure-functions-integration-setup
|
||||
|
||||
- name: Run Integration Tests
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -98,7 +88,6 @@ jobs:
|
||||
env:
|
||||
COSMOSDB_ENDPOINT: https://localhost:8081
|
||||
COSMOSDB_KEY: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
OpenAI__ApiKey: ${{ secrets.OPENAI__APIKEY }}
|
||||
OpenAI__ChatModelId: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OpenAI__ChatReasoningModelId: ${{ vars.OPENAI__CHATREASONINGMODELID }}
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
#
|
||||
# Runs the .NET sample verification tool, which builds and executes sample projects
|
||||
# and verifies their output using deterministic checks and AI-powered verification.
|
||||
#
|
||||
# Results are displayed as a GitHub Job Summary and the CSV report is uploaded as an artifact.
|
||||
#
|
||||
|
||||
name: dotnet-verify-samples
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
category:
|
||||
description: "Sample category to run (blank for all)"
|
||||
required: false
|
||||
type: choice
|
||||
options:
|
||||
- ""
|
||||
- "01-get-started"
|
||||
- "02-agents"
|
||||
- "03-workflows"
|
||||
parallelism:
|
||||
description: "Max parallel sample runs"
|
||||
required: false
|
||||
default: "8"
|
||||
type: string
|
||||
schedule:
|
||||
- cron: "0 6 * * 1-5" # Weekdays at 6:00 UTC
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
verify-samples:
|
||||
runs-on: ubuntu-latest
|
||||
environment: 'integration'
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: |
|
||||
.
|
||||
.github
|
||||
dotnet
|
||||
python
|
||||
declarative-agents
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
|
||||
- name: Azure CLI Login
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Generate filtered solution
|
||||
shell: pwsh
|
||||
run: |
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 `
|
||||
-Solution dotnet/agent-framework-dotnet.slnx `
|
||||
-TargetFramework net10.0 `
|
||||
-Configuration Debug `
|
||||
-OutputPath dotnet/filtered.slnx `
|
||||
-Verbose
|
||||
|
||||
- name: Build solution
|
||||
shell: bash
|
||||
run: dotnet build dotnet/filtered.slnx -f net10.0 --warnaserror
|
||||
|
||||
- name: Run verify-samples
|
||||
id: verify
|
||||
working-directory: dotnet
|
||||
shell: bash
|
||||
run: |
|
||||
CATEGORY_ARG=""
|
||||
if [ -n "$CATEGORY_INPUT" ]; then
|
||||
CATEGORY_ARG="--category $CATEGORY_INPUT"
|
||||
fi
|
||||
|
||||
dotnet run --project eng/verify-samples -- \
|
||||
$CATEGORY_ARG \
|
||||
--parallel "$PARALLELISM" \
|
||||
--md results.md \
|
||||
--csv results.csv \
|
||||
--log results.log
|
||||
env:
|
||||
CATEGORY_INPUT: ${{ github.event.inputs.category || '' }}
|
||||
PARALLELISM: ${{ github.event.inputs.parallelism || '8' }}
|
||||
# OpenAI Models
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
|
||||
OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
|
||||
# Azure OpenAI Models
|
||||
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
|
||||
# Microsoft Foundry
|
||||
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
|
||||
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
|
||||
# Foundry
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
|
||||
- name: Write Job Summary
|
||||
if: always()
|
||||
working-directory: dotnet
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -f results.md ]; then
|
||||
cat results.md >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "⚠️ No results.md generated — verify-samples may have failed to start." >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: verify-samples-results
|
||||
path: |
|
||||
dotnet/results.csv
|
||||
dotnet/results.log
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Fail if samples failed
|
||||
if: always() && steps.verify.outcome == 'failure'
|
||||
shell: bash
|
||||
run: exit 1
|
||||
@@ -1,42 +0,0 @@
|
||||
name: GitHub automation tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/actions/**"
|
||||
- ".github/scripts/**"
|
||||
- ".github/tests/**"
|
||||
- ".github/workflows/python-test-coverage.yml"
|
||||
- ".github/workflows/github-automation-tests.yml"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- ".github/actions/**"
|
||||
- ".github/scripts/**"
|
||||
- ".github/tests/**"
|
||||
- ".github/workflows/python-test-coverage.yml"
|
||||
- ".github/workflows/github-automation-tests.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
||||
with:
|
||||
node-version: "22"
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Run JavaScript tests
|
||||
run: node --test .github/tests/*.js
|
||||
|
||||
- name: Run Python tests
|
||||
run: python .github/tests/test_python_check_coverage.py
|
||||
@@ -3,7 +3,7 @@
|
||||
# Go to Actions → "Integration Tests (Manual)" → Run workflow → enter a PR number or branch name.
|
||||
#
|
||||
# It calls dedicated integration-only workflows (dotnet-integration-tests and python-integration-tests),
|
||||
# passing an immutable commit SHA so they check out and test the approved code.
|
||||
# passing a ref so they check out and test the correct code.
|
||||
# Changed paths are detected here so only the relevant test suites run.
|
||||
#
|
||||
|
||||
@@ -26,6 +26,7 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: integration-tests-manual-${{ github.event.inputs.pr-number || github.event.inputs.branch }}
|
||||
@@ -37,50 +38,67 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
checkout-ref: ${{ steps.resolve.outputs.checkout-ref }}
|
||||
base-ref: ${{ steps.resolve.outputs.base-ref }}
|
||||
dotnet-changes: ${{ steps.detect-changes.outputs.dotnet }}
|
||||
python-changes: ${{ steps.detect-changes.outputs.python }}
|
||||
steps:
|
||||
- name: Check out trusted workflow helpers
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
sparse-checkout: .github/scripts
|
||||
|
||||
- name: Resolve and authorize checkout ref
|
||||
- name: Resolve checkout ref
|
||||
id: resolve
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const resolveIntegrationTestTarget = require(
|
||||
'./.github/scripts/resolve_integration_test_target.js'
|
||||
);
|
||||
const target = await resolveIntegrationTestTarget({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
prNumber: process.env.PR_NUMBER,
|
||||
branch: process.env.BRANCH,
|
||||
});
|
||||
core.setOutput('checkout-ref', target.checkoutRef);
|
||||
core.setOutput('base-ref', target.baseRef);
|
||||
core.info(`Running integration tests for ${target.description} at ${target.checkoutRef}.`);
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.inputs.pr-number }}
|
||||
BRANCH: ${{ github.event.inputs.branch }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
if [ -n "$PR_NUMBER" ] && [ -n "$BRANCH" ]; then
|
||||
echo "::error::Please provide either a PR number or a branch name, not both."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$PR_NUMBER" ] && [ -z "$BRANCH" ]; then
|
||||
echo "::error::Please provide either a PR number or a branch name."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "$PR_NUMBER" ]; then
|
||||
if ! echo "$PR_NUMBER" | grep -Eq '^[0-9]+$'; then
|
||||
echo "::error::Invalid PR number. Only numeric values are allowed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PR_DATA=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json state)
|
||||
PR_STATE=$(echo "$PR_DATA" | jq -r '.state')
|
||||
|
||||
if [ "$PR_STATE" != "OPEN" ]; then
|
||||
echo "::error::PR #$PR_NUMBER is not open (state: $PR_STATE)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "checkout-ref=refs/pull/$PR_NUMBER/head" >> "$GITHUB_OUTPUT"
|
||||
echo "Running integration tests for PR #$PR_NUMBER"
|
||||
else
|
||||
if ! echo "$BRANCH" | grep -Eq '^[a-zA-Z0-9_./-]+$'; then
|
||||
echo "::error::Invalid branch name. Only alphanumeric characters, hyphens, underscores, dots, and slashes are allowed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "checkout-ref=$BRANCH" >> "$GITHUB_OUTPUT"
|
||||
echo "Running integration tests for branch $BRANCH"
|
||||
fi
|
||||
|
||||
- name: Detect changed paths
|
||||
id: detect-changes
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BASE_REF: ${{ steps.resolve.outputs.base-ref }}
|
||||
CHECKOUT_REF: ${{ steps.resolve.outputs.checkout-ref }}
|
||||
PR_NUMBER: ${{ github.event.inputs.pr-number }}
|
||||
BRANCH: ${{ github.event.inputs.branch }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
CHANGED_FILES=$(gh api "repos/$REPO/compare/$BASE_REF...$CHECKOUT_REF" \
|
||||
--jq '.files[].filename')
|
||||
if [ -n "$PR_NUMBER" ]; then
|
||||
CHANGED_FILES=$(gh pr diff "$PR_NUMBER" --repo "$REPO" --name-only)
|
||||
else
|
||||
# For branches, compare against main using the GitHub API
|
||||
CHANGED_FILES=$(gh api "repos/$REPO/compare/main...$BRANCH" --jq '.files[].filename')
|
||||
fi
|
||||
|
||||
DOTNET_CHANGES=false
|
||||
PYTHON_CHANGES=false
|
||||
@@ -95,41 +113,22 @@ jobs:
|
||||
|
||||
echo "dotnet=$DOTNET_CHANGES" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$PYTHON_CHANGES" >> "$GITHUB_OUTPUT"
|
||||
echo "Detected changes; dotnet: $DOTNET_CHANGES, python: $PYTHON_CHANGES"
|
||||
echo "Detected changes — dotnet: $DOTNET_CHANGES, python: $PYTHON_CHANGES"
|
||||
|
||||
dotnet-integration-tests:
|
||||
name: .NET Integration Tests
|
||||
needs: resolve-ref
|
||||
if: needs.resolve-ref.outputs.dotnet-changes == 'true'
|
||||
permissions:
|
||||
copilot-requests: write
|
||||
contents: read
|
||||
id-token: write
|
||||
uses: ./.github/workflows/dotnet-integration-tests.yml
|
||||
with:
|
||||
checkout-ref: ${{ needs.resolve-ref.outputs.checkout-ref }}
|
||||
secrets:
|
||||
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
||||
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
AZUREAI__ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }}
|
||||
OPENAI__APIKEY: ${{ secrets.OPENAI__APIKEY }}
|
||||
secrets: inherit
|
||||
|
||||
python-integration-tests:
|
||||
name: Python Integration Tests
|
||||
needs: resolve-ref
|
||||
if: needs.resolve-ref.outputs.python-changes == 'true'
|
||||
permissions:
|
||||
copilot-requests: write
|
||||
contents: read
|
||||
id-token: write
|
||||
uses: ./.github/workflows/python-integration-tests.yml
|
||||
with:
|
||||
checkout-ref: ${{ needs.resolve-ref.outputs.checkout-ref }}
|
||||
secrets:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
|
||||
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
FOUNDRY_MODELS_API_KEY: ${{ secrets.FOUNDRY_MODELS_API_KEY }}
|
||||
OPENAI__APIKEY: ${{ secrets.OPENAI__APIKEY }}
|
||||
secrets: inherit
|
||||
|
||||
@@ -1,240 +0,0 @@
|
||||
name: Issue Triage
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened, typed]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
issue_number:
|
||||
description: Issue number to triage
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: >-
|
||||
issue-triage-${{ github.repository }}-${{
|
||||
github.event_name == 'workflow_dispatch' && inputs.issue_number
|
||||
|| github.event.issue.type.name == 'Bug' && github.event.issue.number
|
||||
|| github.run_id
|
||||
}}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
DEVFLOW_REPOSITORY: ${{ vars.DF_REPO }}
|
||||
DEVFLOW_REF: main
|
||||
TARGET_REPO_PATH: ${{ github.workspace }}/target-repo
|
||||
DEVFLOW_PATH: ${{ github.workspace }}/devflow
|
||||
MODEL_CONFIG_PATH: ${{ github.workspace }}/devflow/config.ci.yaml
|
||||
|
||||
jobs:
|
||||
team_check:
|
||||
runs-on: ubuntu-latest
|
||||
environment: github-app-auth
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| github.event.issue.type.name == 'Bug'
|
||||
}}
|
||||
outputs:
|
||||
is_team_member: ${{ steps.check.outputs.is_team_member }}
|
||||
issue_number: ${{ steps.issue.outputs.issue_number }}
|
||||
repo: ${{ steps.issue.outputs.repo }}
|
||||
steps:
|
||||
- name: Resolve issue metadata
|
||||
id: issue
|
||||
shell: bash
|
||||
env:
|
||||
ISSUE_NUMBER: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch' && inputs.issue_number
|
||||
|| github.event.issue.number
|
||||
}}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
issue_number="${ISSUE_NUMBER}"
|
||||
|
||||
if [[ ! "$issue_number" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "Could not determine issue number from event payload or manual input." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "issue_number=${issue_number}" >> "$GITHUB_OUTPUT"
|
||||
echo "repo=${GITHUB_REPOSITORY}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout scripts
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/actions/github-app-token
|
||||
.github/scripts
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get GitHub automation token
|
||||
id: github-auth
|
||||
uses: ./.github/actions/github-app-token
|
||||
with:
|
||||
mode: ${{ vars.GH_APP_AUTH_MODE }}
|
||||
azure-client-id: ${{ secrets.GH_APP_AZURE_CLIENT_ID }}
|
||||
azure-tenant-id: ${{ secrets.GH_APP_AZURE_TENANT_ID }}
|
||||
azure-subscription-id: ${{ secrets.GH_APP_AZURE_SUBSCRIPTION_ID }}
|
||||
key-vault-name: ${{ secrets.GH_APP_KEY_VAULT_NAME }}
|
||||
key-name: ${{ secrets.GH_APP_KEY_NAME }}
|
||||
github-app-client-id: ${{ secrets.GH_APP_CLIENT_ID }}
|
||||
github-app-installation-id: ${{ secrets.GH_APP_INSTALLATION_ID }}
|
||||
repository: ${{ github.repository }}
|
||||
fallback-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
|
||||
- name: Check issue author team membership
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
id: check
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
env:
|
||||
TEAM_NAME: ${{ secrets.DEVELOPER_TEAM }}
|
||||
ISSUE_NUMBER: ${{ steps.issue.outputs.issue_number }}
|
||||
with:
|
||||
github-token: ${{ steps.github-auth.outputs.token }}
|
||||
script: |
|
||||
const checkTeamMembership = require('./.github/scripts/check_team_membership.js');
|
||||
const { author, isTeamMember } = await checkTeamMembership({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
teamSlug: process.env.TEAM_NAME,
|
||||
issueNumber: process.env.ISSUE_NUMBER,
|
||||
});
|
||||
core.setOutput('is_team_member', isTeamMember ? 'true' : 'false');
|
||||
if (isTeamMember) {
|
||||
core.info(`Author ${author} is a team member; skipping auto-triage.`);
|
||||
} else {
|
||||
core.info(`Author ${author} is not a team member; proceeding with triage.`);
|
||||
}
|
||||
|
||||
triage:
|
||||
runs-on: ubuntu-latest
|
||||
needs: team_check
|
||||
if: >-
|
||||
${{
|
||||
github.event_name == 'workflow_dispatch'
|
||||
|| needs.team_check.outputs.is_team_member == 'false'
|
||||
}}
|
||||
environment: integration
|
||||
permissions:
|
||||
copilot-requests: write
|
||||
contents: read
|
||||
id-token: write
|
||||
issues: write
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
# Safe checkout: base repo only.
|
||||
- name: Checkout target repo base
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
path: target-repo
|
||||
|
||||
# Private DevFlow (maf-dashboard) checkout.
|
||||
- name: Checkout DevFlow
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
repository: ${{ env.DEVFLOW_REPOSITORY }}
|
||||
ref: ${{ env.DEVFLOW_REF }}
|
||||
token: ${{ secrets.DEVFLOW_TOKEN }}
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
path: devflow
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
version: "0.11.x"
|
||||
enable-cache: true
|
||||
|
||||
- name: Install DevFlow dependencies
|
||||
working-directory: ${{ env.DEVFLOW_PATH }}
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Classify issue relevance
|
||||
id: spam
|
||||
working-directory: ${{ env.DEVFLOW_PATH }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
DEVFLOW_TOKEN: ${{ secrets.DEVFLOW_TOKEN }}
|
||||
SK_REPO_PATH: ${{ env.TARGET_REPO_PATH }}
|
||||
AGENT_REPO_PATH: ${{ env.TARGET_REPO_PATH }}
|
||||
ISSUE_REPO: ${{ needs.team_check.outputs.repo }}
|
||||
ISSUE_NUMBER: ${{ needs.team_check.outputs.issue_number }}
|
||||
run: |
|
||||
uv run python scripts/classify_issue_spam.py \
|
||||
--repo "$ISSUE_REPO" \
|
||||
--issue-number "$ISSUE_NUMBER" \
|
||||
--repo-path "${TARGET_REPO_PATH}" \
|
||||
--apply-labels
|
||||
|
||||
- name: Stop after spam gate
|
||||
if: ${{ steps.spam.outputs.allow_triage != 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Stopping: issue triage preflight did not allow automation."
|
||||
exit 1
|
||||
|
||||
- name: Reproduce reported issue
|
||||
if: ${{ steps.spam.outputs.allow_triage == 'true' }}
|
||||
id: repro
|
||||
working-directory: ${{ env.DEVFLOW_PATH }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
# Not seen by the agent prompt; used only to push a paper-trail
|
||||
# branch back to maf-dashboard at run end.
|
||||
DEVFLOW_TOKEN: ${{ secrets.DEVFLOW_TOKEN }}
|
||||
SK_REPO_PATH: ${{ env.TARGET_REPO_PATH }}
|
||||
AGENT_REPO_PATH: ${{ env.TARGET_REPO_PATH }}
|
||||
ISSUE_REPO: ${{ needs.team_check.outputs.repo }}
|
||||
ISSUE_NUMBER: ${{ needs.team_check.outputs.issue_number }}
|
||||
# Model-provider settings for generated repro code. Never enter the
|
||||
# agent prompt; consumed by SDK constructors via os.environ. Azure
|
||||
# OpenAI and Foundry auth via AAD from the azure/login step above.
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
|
||||
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
|
||||
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME }}
|
||||
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION }}
|
||||
FOUNDRY_MODELS_ENDPOINT: ${{ vars.FOUNDRY_MODELS_ENDPOINT || '' }}
|
||||
FOUNDRY_MODELS_API_KEY: ${{ secrets.FOUNDRY_MODELS_API_KEY || '' }}
|
||||
FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
|
||||
run: |
|
||||
uv run python scripts/trigger_issue_repro.py \
|
||||
--repo "$ISSUE_REPO" \
|
||||
--issue-number "$ISSUE_NUMBER" \
|
||||
--github-username "$GITHUB_ACTOR"
|
||||
@@ -10,39 +10,12 @@ jobs:
|
||||
name: "Issue: add labels"
|
||||
if: ${{ github.event.action == 'opened' || github.event.action == 'reopened' }}
|
||||
runs-on: ubuntu-latest
|
||||
environment: github-app-auth
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
issues: write
|
||||
steps:
|
||||
- name: Checkout GitHub automation
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/github-script@v8
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/actions/github-app-token
|
||||
.github/scripts/check_team_membership.js
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get GitHub automation token
|
||||
id: github-auth
|
||||
uses: ./.github/actions/github-app-token
|
||||
with:
|
||||
mode: ${{ vars.GH_APP_AUTH_MODE }}
|
||||
azure-client-id: ${{ secrets.GH_APP_AZURE_CLIENT_ID }}
|
||||
azure-tenant-id: ${{ secrets.GH_APP_AZURE_TENANT_ID }}
|
||||
azure-subscription-id: ${{ secrets.GH_APP_AZURE_SUBSCRIPTION_ID }}
|
||||
key-vault-name: ${{ secrets.GH_APP_KEY_VAULT_NAME }}
|
||||
key-name: ${{ secrets.GH_APP_KEY_NAME }}
|
||||
github-app-client-id: ${{ secrets.GH_APP_CLIENT_ID }}
|
||||
github-app-installation-id: ${{ secrets.GH_APP_INSTALLATION_ID }}
|
||||
repository: ${{ github.repository }}
|
||||
fallback-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
|
||||
- uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ steps.github-auth.outputs.token }}
|
||||
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
script: |
|
||||
// Get the issue body and title
|
||||
const body = context.payload.issue.body
|
||||
@@ -51,14 +24,21 @@ jobs:
|
||||
// Define the labels array
|
||||
let labels = []
|
||||
|
||||
const checkTeamMembership = require('./.github/scripts/check_team_membership.js')
|
||||
const { isTeamMember } = await checkTeamMembership({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
teamSlug: process.env.TEAM_NAME,
|
||||
issueNumber: context.issue.number,
|
||||
})
|
||||
// Check if the issue author is in the agentframework-developers team
|
||||
let isTeamMember = false
|
||||
try {
|
||||
const teamMembership = await github.rest.teams.getMembershipForUserInOrg({
|
||||
org: context.repo.owner,
|
||||
team_slug: process.env.TEAM_NAME,
|
||||
username: context.payload.issue.user.login
|
||||
})
|
||||
console.log("Team Membership Data:", teamMembership);
|
||||
isTeamMember = teamMembership.data.state === 'active'
|
||||
} catch (error) {
|
||||
// User is not in the team or team doesn't exist
|
||||
console.error("Error fetching team membership:", error);
|
||||
isTeamMember = false
|
||||
}
|
||||
|
||||
// Only add triage label if the author is not in the team
|
||||
if (!isTeamMember) {
|
||||
@@ -110,7 +90,9 @@ jobs:
|
||||
// Check for issue type from issue form dropdown
|
||||
const issueTypeField = getFormFieldValue(body, 'Type of Issue')
|
||||
if (issueTypeField) {
|
||||
if (issueTypeField === 'Feature Request') {
|
||||
if (issueTypeField === 'Bug') {
|
||||
labels.push("bug")
|
||||
} else if (issueTypeField === 'Feature Request') {
|
||||
labels.push("enhancement")
|
||||
} else if (issueTypeField === 'Question') {
|
||||
labels.push("question")
|
||||
|
||||
@@ -6,54 +6,16 @@
|
||||
# https://github.com/actions/labeler
|
||||
|
||||
name: Label pull request
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, edited]
|
||||
on: [pull_request_target]
|
||||
|
||||
jobs:
|
||||
add_label:
|
||||
runs-on: ubuntu-latest
|
||||
environment: github-app-auth
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout scripts
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/labeler@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
sparse-checkout: |
|
||||
.github/actions/github-app-token
|
||||
.github/scripts
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get GitHub automation token
|
||||
id: github-auth
|
||||
uses: ./.github/actions/github-app-token
|
||||
with:
|
||||
mode: ${{ vars.GH_APP_AUTH_MODE }}
|
||||
azure-client-id: ${{ secrets.GH_APP_AZURE_CLIENT_ID }}
|
||||
azure-tenant-id: ${{ secrets.GH_APP_AZURE_TENANT_ID }}
|
||||
azure-subscription-id: ${{ secrets.GH_APP_AZURE_SUBSCRIPTION_ID }}
|
||||
key-vault-name: ${{ secrets.GH_APP_KEY_VAULT_NAME }}
|
||||
key-name: ${{ secrets.GH_APP_KEY_NAME }}
|
||||
github-app-client-id: ${{ secrets.GH_APP_CLIENT_ID }}
|
||||
github-app-installation-id: ${{ secrets.GH_APP_INSTALLATION_ID }}
|
||||
repository: ${{ github.repository }}
|
||||
fallback-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
|
||||
- uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6
|
||||
with:
|
||||
repo-token: ${{ steps.github-auth.outputs.token }}
|
||||
|
||||
- name: "PR: add breaking change label from title"
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ steps.github-auth.outputs.token }}
|
||||
script: |
|
||||
const { syncBreakingChangeLabelFromTitle } = require('./.github/scripts/title_prefix.js');
|
||||
await syncBreakingChangeLabelFromTitle({ github, context, core });
|
||||
repo-token: "${{ secrets.GH_ACTIONS_PR_WRITE }}"
|
||||
|
||||
@@ -15,17 +15,58 @@ jobs:
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout scripts
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
- uses: actions/github-script@v8
|
||||
name: "Issue/PR: update title"
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const { updateTitleForAddedLabel } = require('./.github/scripts/title_prefix.js');
|
||||
await updateTitleForAddedLabel({ github, context, core });
|
||||
let prefixLabels = {
|
||||
"python": "Python",
|
||||
".NET": ".NET"
|
||||
};
|
||||
|
||||
function addTitlePrefix(title, prefix)
|
||||
{
|
||||
// Update the title based on the label and prefix
|
||||
// Check if the title starts with the prefix (case-sensitive)
|
||||
if (!title.startsWith(prefix + ": ")) {
|
||||
// If not, check if the first word is the label (case-insensitive)
|
||||
if (title.match(new RegExp(`^${prefix}`, 'i'))) {
|
||||
// If yes, replace it with the prefix (case-sensitive)
|
||||
title = title.replace(new RegExp(`^${prefix}`, 'i'), prefix);
|
||||
} else {
|
||||
// If not, prepend the prefix to the title
|
||||
title = prefix + ": " + title;
|
||||
}
|
||||
}
|
||||
|
||||
return title;
|
||||
}
|
||||
|
||||
labelAdded = context.payload.label.name
|
||||
|
||||
// Check if the issue or PR has the label
|
||||
if (labelAdded in prefixLabels) {
|
||||
let prefix = prefixLabels[labelAdded];
|
||||
switch(context.eventName) {
|
||||
case 'issues':
|
||||
github.rest.issues.update({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: addTitlePrefix(context.payload.issue.title, prefix)
|
||||
});
|
||||
break
|
||||
|
||||
case 'pull_request_target':
|
||||
github.rest.pulls.update({
|
||||
pull_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: addTitlePrefix(context.payload.pull_request.title, prefix)
|
||||
});
|
||||
break
|
||||
default:
|
||||
core.setFailed('Unrecognited eventName: ' + context.eventName);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
name: Limit community pull requests
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: pr-limit-${{ github.repository }}-${{ github.event.pull_request.user.login }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
MAX_OPEN_PULL_REQUESTS: '10'
|
||||
PR_LIMIT_EXEMPT_LABEL: pr-limit-exempt
|
||||
TOO_MANY_PRS_LABEL: too-many-prs
|
||||
|
||||
jobs:
|
||||
team_check:
|
||||
runs-on: ubuntu-latest
|
||||
environment: github-app-auth
|
||||
outputs:
|
||||
is_team_member: ${{ steps.check.outputs.is_team_member }}
|
||||
steps:
|
||||
- name: Checkout scripts
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
sparse-checkout: |
|
||||
.github/actions/github-app-token
|
||||
.github/scripts
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get GitHub automation token
|
||||
id: github-auth
|
||||
uses: ./.github/actions/github-app-token
|
||||
with:
|
||||
mode: ${{ vars.GH_APP_AUTH_MODE }}
|
||||
azure-client-id: ${{ secrets.GH_APP_AZURE_CLIENT_ID }}
|
||||
azure-tenant-id: ${{ secrets.GH_APP_AZURE_TENANT_ID }}
|
||||
azure-subscription-id: ${{ secrets.GH_APP_AZURE_SUBSCRIPTION_ID }}
|
||||
key-vault-name: ${{ secrets.GH_APP_KEY_VAULT_NAME }}
|
||||
key-name: ${{ secrets.GH_APP_KEY_NAME }}
|
||||
github-app-client-id: ${{ secrets.GH_APP_CLIENT_ID }}
|
||||
github-app-installation-id: ${{ secrets.GH_APP_INSTALLATION_ID }}
|
||||
repository: ${{ github.repository }}
|
||||
fallback-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
|
||||
- name: Check PR author team membership
|
||||
id: check
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
env:
|
||||
TEAM_NAME: ${{ secrets.DEVELOPER_TEAM }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
with:
|
||||
github-token: ${{ steps.github-auth.outputs.token }}
|
||||
script: |
|
||||
const checkTeamMembership = require('./.github/scripts/check_team_membership.js');
|
||||
const { author, isTeamMember } = await checkTeamMembership({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
teamSlug: process.env.TEAM_NAME,
|
||||
issueNumber: process.env.PR_NUMBER,
|
||||
});
|
||||
core.setOutput('is_team_member', isTeamMember ? 'true' : 'false');
|
||||
if (isTeamMember) {
|
||||
core.info(`Author ${author} is a team member; skipping open PR limit.`);
|
||||
} else {
|
||||
core.info(`Author ${author} is not a team member; checking open PR limit.`);
|
||||
}
|
||||
|
||||
limit_open_prs:
|
||||
runs-on: ubuntu-latest
|
||||
environment: github-app-auth
|
||||
needs: team_check
|
||||
if: ${{ needs.team_check.outputs.is_team_member == 'false' }}
|
||||
steps:
|
||||
- name: Checkout scripts
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
sparse-checkout: |
|
||||
.github/actions/github-app-token
|
||||
.github/scripts
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get GitHub automation token
|
||||
id: github-auth
|
||||
uses: ./.github/actions/github-app-token
|
||||
with:
|
||||
mode: ${{ vars.GH_APP_AUTH_MODE }}
|
||||
azure-client-id: ${{ secrets.GH_APP_AZURE_CLIENT_ID }}
|
||||
azure-tenant-id: ${{ secrets.GH_APP_AZURE_TENANT_ID }}
|
||||
azure-subscription-id: ${{ secrets.GH_APP_AZURE_SUBSCRIPTION_ID }}
|
||||
key-vault-name: ${{ secrets.GH_APP_KEY_VAULT_NAME }}
|
||||
key-name: ${{ secrets.GH_APP_KEY_NAME }}
|
||||
github-app-client-id: ${{ secrets.GH_APP_CLIENT_ID }}
|
||||
github-app-installation-id: ${{ secrets.GH_APP_INSTALLATION_ID }}
|
||||
repository: ${{ github.repository }}
|
||||
fallback-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
|
||||
- name: Enforce open PR limit
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ steps.github-auth.outputs.token }}
|
||||
script: |
|
||||
const { enforcePrLimit } = require('./.github/scripts/pr_limit_moderation.js');
|
||||
await enforcePrLimit({
|
||||
github,
|
||||
context,
|
||||
core,
|
||||
exemptLabelName: process.env.PR_LIMIT_EXEMPT_LABEL,
|
||||
maxOpenPrs: Number.parseInt(process.env.MAX_OPEN_PULL_REQUESTS, 10),
|
||||
labelName: process.env.TOO_MANY_PRS_LABEL,
|
||||
});
|
||||
@@ -19,21 +19,13 @@ jobs:
|
||||
runs-on: ubuntu-22.04
|
||||
# check out the latest version of the code
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
- name: Install Chrome for Puppeteer
|
||||
run: npx puppeteer browsers install chrome
|
||||
|
||||
# Checks the status of hyperlinks in all files
|
||||
- name: Run linkspector
|
||||
uses: umbrelladocs/action-linkspector@963b6264d7de32c904942a70b488d3407453049e # v1
|
||||
uses: umbrelladocs/action-linkspector@v1
|
||||
with:
|
||||
reporter: local
|
||||
filter_mode: nofilter
|
||||
|
||||
@@ -2,7 +2,7 @@ name: Merge Gatekeeper
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["main", "feature*"]
|
||||
branches: [ "main", "feature*" ]
|
||||
merge_group:
|
||||
branches: ["main"]
|
||||
|
||||
@@ -13,105 +13,23 @@ concurrency:
|
||||
jobs:
|
||||
merge-gatekeeper:
|
||||
runs-on: ubuntu-latest
|
||||
# Restrict permissions of the GITHUB_TOKEN.
|
||||
# Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
|
||||
permissions:
|
||||
checks: read
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Wait for required checks
|
||||
- name: Run Merge Gatekeeper
|
||||
# NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
|
||||
# https://github.com/upsidr/merge-gatekeeper/tags
|
||||
# https://github.com/upsidr/merge-gatekeeper/branches
|
||||
uses: upsidr/merge-gatekeeper@v1
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
env:
|
||||
TIMEOUT_SECONDS: "3600"
|
||||
INTERVAL_SECONDS: "30"
|
||||
SELF_JOB_NAME: ${{ github.job }}
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
timeout: 3600
|
||||
interval: 30
|
||||
# "Cleanup artifacts", "Agent", "Prepare", and "Upload results" are check runs
|
||||
# created by an org-level GitHub App (MSDO), not by any workflow in this repo.
|
||||
# They are outside our control and their transient failures should not block merges.
|
||||
IGNORED_NAMES: "CodeQL,CodeQL analysis (csharp),Cleanup artifacts,Agent,Prepare,Upload results,review"
|
||||
with:
|
||||
script: |
|
||||
const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS);
|
||||
const intervalSeconds = Number(process.env.INTERVAL_SECONDS);
|
||||
const selfName = process.env.SELF_JOB_NAME;
|
||||
const ignored = new Set(
|
||||
process.env.IGNORED_NAMES.split(',').map((s) => s.trim()).filter(Boolean),
|
||||
);
|
||||
|
||||
const sha = context.payload.pull_request.head.sha;
|
||||
const { owner, repo } = context.repo;
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
// Mirrors upsidr/merge-gatekeeper: merge combined-statuses and check-runs
|
||||
// for the PR head SHA, with combined-statuses winning on name collision.
|
||||
async function collectChecks() {
|
||||
const merged = new Map();
|
||||
|
||||
const combined = await github.rest.repos.getCombinedStatusForRef({
|
||||
owner, repo, ref: sha, per_page: 100,
|
||||
});
|
||||
for (const s of combined.data.statuses ?? []) {
|
||||
if (!merged.has(s.context)) {
|
||||
// Combined-status states: success | pending | error | failure
|
||||
merged.set(s.context, { name: s.context, state: s.state });
|
||||
}
|
||||
}
|
||||
|
||||
const runs = await github.paginate(github.rest.checks.listForRef, {
|
||||
owner, repo, ref: sha, per_page: 100,
|
||||
});
|
||||
for (const r of runs) {
|
||||
if (merged.has(r.name)) continue;
|
||||
let state;
|
||||
if (r.status !== 'completed') {
|
||||
state = 'pending';
|
||||
} else if (r.conclusion === 'skipped') {
|
||||
continue; // Skipped runs are dropped, matching the original action.
|
||||
} else if (r.conclusion === 'success' || r.conclusion === 'neutral') {
|
||||
state = 'success';
|
||||
} else {
|
||||
// cancelled | timed_out | action_required | stale | failure
|
||||
state = 'error';
|
||||
}
|
||||
merged.set(r.name, { name: r.name, state });
|
||||
}
|
||||
|
||||
return [...merged.values()];
|
||||
}
|
||||
|
||||
function evaluate(entries) {
|
||||
const failed = [];
|
||||
const pending = [];
|
||||
const succeeded = [];
|
||||
for (const e of entries) {
|
||||
if (e.name === selfName || ignored.has(e.name)) continue;
|
||||
if (e.state === 'success') succeeded.push(e.name);
|
||||
else if (e.state === 'error' || e.state === 'failure') failed.push(e.name);
|
||||
else pending.push(e.name);
|
||||
}
|
||||
return { failed, pending, succeeded };
|
||||
}
|
||||
|
||||
const deadline = Date.now() + timeoutSeconds * 1000;
|
||||
for (;;) {
|
||||
const entries = await collectChecks();
|
||||
const { failed, pending, succeeded } = evaluate(entries);
|
||||
|
||||
core.info(
|
||||
`succeeded=${succeeded.length} pending=${pending.length} failed=${failed.length}`,
|
||||
);
|
||||
if (failed.length) {
|
||||
core.setFailed(`Failing checks: ${failed.join(', ')}`);
|
||||
return;
|
||||
}
|
||||
if (pending.length === 0) {
|
||||
core.info(`All required checks passed: ${succeeded.join(', ') || '(none)'}`);
|
||||
return;
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
core.setFailed(`Timed out waiting for: ${pending.join(', ')}`);
|
||||
return;
|
||||
}
|
||||
core.info(`Waiting on (${pending.length}): ${pending.slice(0, 10).join(', ')}${pending.length > 10 ? ', …' : ''}`);
|
||||
await sleep(intervalSeconds * 1000);
|
||||
}
|
||||
ignored: CodeQL,CodeQL analysis (csharp),Cleanup artifacts,Agent,Prepare,Upload results
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
"""Check Python test coverage against threshold for enforced targets.
|
||||
|
||||
This script parses a Cobertura XML coverage report and enforces a minimum
|
||||
coverage threshold on specific targets. Targets can be package names
|
||||
(e.g., "packages.core.agent_framework") or individual Python file paths
|
||||
(e.g., "packages/core/agent_framework/observability.py").
|
||||
|
||||
Non-enforced targets are reported for visibility but don't block the build.
|
||||
|
||||
Usage:
|
||||
python python-check-coverage.py <coverage-xml-path> <threshold>
|
||||
|
||||
Example:
|
||||
python python-check-coverage.py python-coverage.xml 85
|
||||
"""
|
||||
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
|
||||
# =============================================================================
|
||||
# ENFORCED TARGETS CONFIGURATION
|
||||
# =============================================================================
|
||||
# Add or remove entries from this set to control which targets must meet
|
||||
# the coverage threshold. Only these targets will fail the build if below
|
||||
# threshold. Other targets are reported for visibility only.
|
||||
#
|
||||
# Target values can be:
|
||||
# - Package paths as they appear in the coverage report
|
||||
# (e.g., "packages.azure-ai.agent_framework_azure_ai")
|
||||
# - Python source file paths as they appear in the coverage report
|
||||
# (e.g., "packages/core/agent_framework/observability.py")
|
||||
# =============================================================================
|
||||
ENFORCED_TARGETS: set[str] = {
|
||||
# Packages
|
||||
"packages.azure-ai.agent_framework_azure_ai",
|
||||
"packages.core.agent_framework",
|
||||
"packages.core.agent_framework._workflows",
|
||||
"packages.purview.agent_framework_purview",
|
||||
"packages.anthropic.agent_framework_anthropic",
|
||||
"packages.azure-ai-search.agent_framework_azure_ai_search",
|
||||
"packages.openai.agent_framework_openai",
|
||||
# Individual files (if you want to enforce specific files instead of whole packages)
|
||||
"packages/core/agent_framework/observability.py",
|
||||
# Add more targets here as coverage improves
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackageCoverage:
|
||||
"""Coverage data for a single package."""
|
||||
|
||||
name: str
|
||||
line_rate: float
|
||||
branch_rate: float
|
||||
lines_valid: int
|
||||
lines_covered: int
|
||||
branches_valid: int
|
||||
branches_covered: int
|
||||
|
||||
@property
|
||||
def line_coverage_percent(self) -> float:
|
||||
"""Return line coverage as a percentage."""
|
||||
return self.line_rate * 100
|
||||
|
||||
@property
|
||||
def branch_coverage_percent(self) -> float:
|
||||
"""Return branch coverage as a percentage."""
|
||||
return self.branch_rate * 100
|
||||
|
||||
|
||||
def normalize_coverage_path(path: str) -> str:
|
||||
"""Normalize coverage paths for reliable matching."""
|
||||
return path.replace("\\", "/").lstrip("./")
|
||||
|
||||
|
||||
def parse_coverage_xml(
|
||||
xml_path: str,
|
||||
) -> tuple[dict[str, PackageCoverage], dict[str, PackageCoverage], float, float]:
|
||||
"""Parse Cobertura XML and extract per-package coverage data.
|
||||
|
||||
Args:
|
||||
xml_path: Path to the Cobertura XML coverage report.
|
||||
|
||||
Returns:
|
||||
A tuple of (packages_dict, files_dict, overall_line_rate, overall_branch_rate).
|
||||
"""
|
||||
tree = ET.parse(xml_path)
|
||||
root = tree.getroot()
|
||||
|
||||
# Get overall coverage from root element
|
||||
overall_line_rate = float(root.get("line-rate", 0))
|
||||
overall_branch_rate = float(root.get("branch-rate", 0))
|
||||
|
||||
packages: dict[str, PackageCoverage] = {}
|
||||
file_stats: dict[str, dict[str, int]] = {}
|
||||
|
||||
for package in root.findall(".//package"):
|
||||
package_path = package.get("name", "unknown")
|
||||
|
||||
line_rate = float(package.get("line-rate", 0))
|
||||
branch_rate = float(package.get("branch-rate", 0))
|
||||
|
||||
# Count lines and branches from classes within this package
|
||||
lines_valid = 0
|
||||
lines_covered = 0
|
||||
branches_valid = 0
|
||||
branches_covered = 0
|
||||
|
||||
for class_elem in package.findall(".//class"):
|
||||
file_path = normalize_coverage_path(class_elem.get("filename", ""))
|
||||
if file_path and file_path not in file_stats:
|
||||
file_stats[file_path] = {
|
||||
"lines_valid": 0,
|
||||
"lines_covered": 0,
|
||||
"branches_valid": 0,
|
||||
"branches_covered": 0,
|
||||
}
|
||||
|
||||
for line in class_elem.findall(".//line"):
|
||||
lines_valid += 1
|
||||
if int(line.get("hits", 0)) > 0:
|
||||
lines_covered += 1
|
||||
|
||||
if file_path:
|
||||
file_stats[file_path]["lines_valid"] += 1
|
||||
if int(line.get("hits", 0)) > 0:
|
||||
file_stats[file_path]["lines_covered"] += 1
|
||||
|
||||
# Branch coverage from line elements
|
||||
if line.get("branch") == "true":
|
||||
condition_coverage = line.get("condition-coverage", "")
|
||||
if condition_coverage:
|
||||
# Parse "X% (covered/total)" format
|
||||
try:
|
||||
coverage_parts = (
|
||||
condition_coverage.split("(")[1].rstrip(")").split("/")
|
||||
)
|
||||
branches_covered += int(coverage_parts[0])
|
||||
branches_valid += int(coverage_parts[1])
|
||||
if file_path:
|
||||
file_stats[file_path]["branches_covered"] += int(
|
||||
coverage_parts[0]
|
||||
)
|
||||
file_stats[file_path]["branches_valid"] += int(
|
||||
coverage_parts[1]
|
||||
)
|
||||
except (IndexError, ValueError):
|
||||
# Ignore malformed condition-coverage strings; treat this line as having no branch data.
|
||||
pass
|
||||
|
||||
# Use full package path as the key (no aggregation)
|
||||
packages[package_path] = PackageCoverage(
|
||||
name=package_path,
|
||||
line_rate=line_rate if lines_valid == 0 else lines_covered / lines_valid,
|
||||
branch_rate=branch_rate
|
||||
if branches_valid == 0
|
||||
else branches_covered / branches_valid,
|
||||
lines_valid=lines_valid,
|
||||
lines_covered=lines_covered,
|
||||
branches_valid=branches_valid,
|
||||
branches_covered=branches_covered,
|
||||
)
|
||||
|
||||
files: dict[str, PackageCoverage] = {}
|
||||
for file_path, stats in file_stats.items():
|
||||
lines_valid = stats["lines_valid"]
|
||||
lines_covered = stats["lines_covered"]
|
||||
branches_valid = stats["branches_valid"]
|
||||
branches_covered = stats["branches_covered"]
|
||||
|
||||
files[file_path] = PackageCoverage(
|
||||
name=file_path,
|
||||
line_rate=0 if lines_valid == 0 else lines_covered / lines_valid,
|
||||
branch_rate=0 if branches_valid == 0 else branches_covered / branches_valid,
|
||||
lines_valid=lines_valid,
|
||||
lines_covered=lines_covered,
|
||||
branches_valid=branches_valid,
|
||||
branches_covered=branches_covered,
|
||||
)
|
||||
|
||||
return packages, files, overall_line_rate, overall_branch_rate
|
||||
|
||||
|
||||
def format_coverage_value(coverage: float, threshold: float, is_enforced: bool) -> str:
|
||||
"""Format a coverage value with optional pass/fail indicator.
|
||||
|
||||
Args:
|
||||
coverage: Coverage percentage (0-100).
|
||||
threshold: Minimum required coverage percentage.
|
||||
is_enforced: Whether this target is enforced.
|
||||
|
||||
Returns:
|
||||
Formatted string like "85.5%" or "85.5% ✅" or "75.0% ❌".
|
||||
"""
|
||||
formatted = f"{coverage:.1f}%"
|
||||
if is_enforced:
|
||||
icon = "✅" if coverage >= threshold else "❌"
|
||||
formatted = f"{formatted} {icon}"
|
||||
return formatted
|
||||
|
||||
|
||||
def print_coverage_table(
|
||||
packages: dict[str, PackageCoverage],
|
||||
files: dict[str, PackageCoverage],
|
||||
threshold: float,
|
||||
overall_line_rate: float,
|
||||
overall_branch_rate: float,
|
||||
) -> None:
|
||||
"""Print a formatted coverage summary table.
|
||||
|
||||
Args:
|
||||
packages: Dictionary of package name to coverage data.
|
||||
files: Dictionary of file path to coverage data, used for per-file enforcement.
|
||||
threshold: Minimum required coverage percentage.
|
||||
overall_line_rate: Overall line coverage rate (0-1).
|
||||
overall_branch_rate: Overall branch coverage rate (0-1).
|
||||
"""
|
||||
print("\n" + "=" * 80)
|
||||
print("PYTHON TEST COVERAGE REPORT")
|
||||
print("=" * 80)
|
||||
|
||||
# Overall coverage
|
||||
print(f"\nOverall Line Coverage: {overall_line_rate * 100:.1f}%")
|
||||
print(f"Overall Branch Coverage: {overall_branch_rate * 100:.1f}%")
|
||||
print(f"Threshold: {threshold}%")
|
||||
|
||||
enforced_targets = {normalize_coverage_path(t) for t in ENFORCED_TARGETS}
|
||||
|
||||
# Package table
|
||||
print("\n" + "-" * 110)
|
||||
print(f"{'Package':<80} {'Lines':<15} {'Line Cov':<15}")
|
||||
print("-" * 110)
|
||||
|
||||
# Sort: enforced package targets first, then alphabetically
|
||||
sorted_packages = sorted(
|
||||
packages.values(),
|
||||
key=lambda p: (p.name not in ENFORCED_TARGETS, p.name),
|
||||
)
|
||||
|
||||
for pkg in sorted_packages:
|
||||
is_enforced = normalize_coverage_path(pkg.name) in enforced_targets
|
||||
enforced_marker = "[ENFORCED] " if is_enforced else ""
|
||||
line_cov = format_coverage_value(
|
||||
pkg.line_coverage_percent, threshold, is_enforced
|
||||
)
|
||||
lines_info = f"{pkg.lines_covered}/{pkg.lines_valid}"
|
||||
package_label = f"{enforced_marker}{pkg.name}"
|
||||
|
||||
print(f"{package_label:<80} {lines_info:<15} {line_cov:<15}")
|
||||
|
||||
print("-" * 110)
|
||||
|
||||
# Enforced file/model entries (if configured)
|
||||
enforced_files = [
|
||||
files[target]
|
||||
for target in sorted(enforced_targets)
|
||||
if target in files and target.endswith(".py")
|
||||
]
|
||||
|
||||
if enforced_files:
|
||||
print("\nEnforced Files/Models")
|
||||
print("-" * 110)
|
||||
print(f"{'File':<80} {'Lines':<15} {'Line Cov':<15}")
|
||||
print("-" * 110)
|
||||
|
||||
for file_cov in enforced_files:
|
||||
line_cov = format_coverage_value(
|
||||
file_cov.line_coverage_percent, threshold, True
|
||||
)
|
||||
lines_info = f"{file_cov.lines_covered}/{file_cov.lines_valid}"
|
||||
print(f"[ENFORCED] {file_cov.name:<69} {lines_info:<15} {line_cov:<15}")
|
||||
|
||||
print("-" * 110)
|
||||
|
||||
|
||||
def check_coverage(xml_path: str, threshold: float) -> bool:
|
||||
"""Check if all enforced targets meet the coverage threshold.
|
||||
|
||||
Args:
|
||||
xml_path: Path to the Cobertura XML coverage report.
|
||||
threshold: Minimum required coverage percentage.
|
||||
|
||||
Returns:
|
||||
True if all enforced targets pass, False otherwise.
|
||||
"""
|
||||
packages, files, overall_line_rate, overall_branch_rate = parse_coverage_xml(
|
||||
xml_path
|
||||
)
|
||||
|
||||
print_coverage_table(
|
||||
packages, files, threshold, overall_line_rate, overall_branch_rate
|
||||
)
|
||||
|
||||
# Check enforced targets
|
||||
failed_targets: list[str] = []
|
||||
missing_targets: list[str] = []
|
||||
|
||||
for target_name in ENFORCED_TARGETS:
|
||||
normalized_target = normalize_coverage_path(target_name)
|
||||
package_alias = normalized_target.replace("/", ".")
|
||||
|
||||
target_coverage = None
|
||||
if target_name in packages:
|
||||
target_coverage = packages[target_name]
|
||||
elif normalized_target in files:
|
||||
target_coverage = files[normalized_target]
|
||||
elif package_alias in packages:
|
||||
target_coverage = packages[package_alias]
|
||||
|
||||
if target_coverage is None:
|
||||
missing_targets.append(target_name)
|
||||
continue
|
||||
|
||||
if target_coverage.line_coverage_percent < threshold:
|
||||
failed_targets.append(
|
||||
f"{target_name} ({target_coverage.line_coverage_percent:.1f}%)"
|
||||
)
|
||||
|
||||
# Report results
|
||||
if missing_targets:
|
||||
print(
|
||||
f"\n❌ FAILED: Enforced targets not found in coverage report: {', '.join(missing_targets)}"
|
||||
)
|
||||
return False
|
||||
|
||||
if failed_targets:
|
||||
print(
|
||||
f"\n❌ FAILED: The following enforced targets are below {threshold}% coverage threshold:"
|
||||
)
|
||||
for target in failed_targets:
|
||||
print(f" - {target}")
|
||||
print("\nTo fix: Add more tests to improve coverage for the failing targets.")
|
||||
return False
|
||||
|
||||
if ENFORCED_TARGETS:
|
||||
found_enforced = [
|
||||
target
|
||||
for target in ENFORCED_TARGETS
|
||||
if target in packages or normalize_coverage_path(target) in files
|
||||
]
|
||||
if found_enforced:
|
||||
print(
|
||||
f"\n✅ PASSED: All enforced targets meet the {threshold}% coverage threshold."
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Main entry point.
|
||||
|
||||
Returns:
|
||||
Exit code: 0 for success, 1 for failure.
|
||||
"""
|
||||
if len(sys.argv) != 3:
|
||||
print(f"Usage: {sys.argv[0]} <coverage-xml-path> <threshold>")
|
||||
print(f"Example: {sys.argv[0]} python-coverage.xml 85")
|
||||
return 1
|
||||
|
||||
xml_path = sys.argv[1]
|
||||
try:
|
||||
threshold = float(sys.argv[2])
|
||||
except ValueError:
|
||||
print(f"Error: Invalid threshold value: {sys.argv[2]}")
|
||||
return 1
|
||||
|
||||
try:
|
||||
success = check_coverage(xml_path, threshold)
|
||||
return 0 if success else 1
|
||||
except FileNotFoundError:
|
||||
print(f"Error: Coverage file not found: {xml_path}")
|
||||
return 1
|
||||
except ET.ParseError as e:
|
||||
print(f"Error: Failed to parse coverage XML: {e}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -6,10 +6,6 @@ on:
|
||||
branches: ["main"]
|
||||
paths:
|
||||
- "python/**"
|
||||
- "!python/AGENTS.md"
|
||||
- "!python/**/AGENTS.md"
|
||||
- "!python/.github/skills/*"
|
||||
- "!python/.github/skills/**"
|
||||
|
||||
env:
|
||||
# Configure a constant location for the uv cache
|
||||
@@ -31,7 +27,7 @@ jobs:
|
||||
env:
|
||||
UV_PYTHON: ${{ matrix.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up python and install the project
|
||||
@@ -42,11 +38,11 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
|
||||
- uses: actions/cache@v5
|
||||
with:
|
||||
path: ~/.cache/prek
|
||||
key: prek|${{ matrix.python-version }}|${{ hashFiles('python/.pre-commit-config.yaml') }}
|
||||
- uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4
|
||||
- uses: j178/prek-action@v1
|
||||
name: Run Pre-commit Hooks (excluding poe-check)
|
||||
env:
|
||||
SKIP: poe-check
|
||||
@@ -68,7 +64,7 @@ jobs:
|
||||
env:
|
||||
UV_PYTHON: ${{ matrix.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up python and install the project
|
||||
@@ -97,7 +93,7 @@ jobs:
|
||||
env:
|
||||
UV_PYTHON: ${{ matrix.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up python and install the project
|
||||
@@ -113,8 +109,8 @@ jobs:
|
||||
- name: Run markdown code lint
|
||||
run: uv run poe markdown-code-lint
|
||||
|
||||
test-typing:
|
||||
name: Test Typing Checks
|
||||
mypy:
|
||||
name: Mypy Checks
|
||||
if: "!cancelled()"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -128,7 +124,7 @@ jobs:
|
||||
env:
|
||||
UV_PYTHON: ${{ matrix.python-version }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up python and install the project
|
||||
@@ -139,5 +135,7 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
- name: Run tests/samples type checkers (mypy, pyrefly, ty)
|
||||
run: uv run python scripts/workspace_poe_tasks.py ci-test-typing
|
||||
- name: Run Mypy
|
||||
env:
|
||||
GITHUB_BASE_REF: ${{ github.event.pull_request.base.ref || github.base_ref || 'main' }}
|
||||
run: uv run python scripts/workspace_poe_tasks.py ci-mypy
|
||||
|
||||
@@ -1,431 +0,0 @@
|
||||
name: Python - Dependency Maintenance
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 4 * * 1"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: python-dependency-maintenance
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
dependency-maintenance:
|
||||
name: Dependency Maintenance
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Match the existing Python dependency maintenance workflows. Reevaluate if package
|
||||
# installability starts differing across supported Python versions.
|
||||
UV_PYTHON: "3.13"
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Set dependency release cutoff
|
||||
run: |
|
||||
cutoff="$(date -u -d '7 days ago' '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
echo "DEPENDENCY_RELEASE_CUTOFF=${cutoff}" >> "$GITHUB_ENV"
|
||||
echo "Using dependency release cutoff: ${cutoff}"
|
||||
|
||||
- name: Repin dev dependency declarations
|
||||
run: uv run poe upgrade-dev-dependency-pins
|
||||
working-directory: ./python
|
||||
|
||||
- name: Refresh lockfile after dev pin updates
|
||||
run: uv lock
|
||||
working-directory: ./python
|
||||
|
||||
- name: Save dev dependency changes
|
||||
run: |
|
||||
DEV_PATCH="${RUNNER_TEMP}/python-dev-dependency-updates.patch"
|
||||
git diff -- python/pyproject.toml "python/packages/*/pyproject.toml" python/uv.lock > "${DEV_PATCH}"
|
||||
if [ -s "${DEV_PATCH}" ]; then
|
||||
echo "has_dev_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_dev_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "patch=${DEV_PATCH}" >> "$GITHUB_OUTPUT"
|
||||
id: dev_changes
|
||||
|
||||
- name: Run dependency bounds test scenarios
|
||||
id: validate_bounds_test
|
||||
continue-on-error: true
|
||||
run: uv run poe validate-dependency-bounds-test --package "*"
|
||||
working-directory: ./python
|
||||
|
||||
- name: Run dependency upper-bound validation
|
||||
id: validate_ranges
|
||||
if: steps.validate_bounds_test.outcome == 'success'
|
||||
continue-on-error: true
|
||||
run: uv run poe validate-dependency-bounds-project --mode upper --package "*"
|
||||
working-directory: ./python
|
||||
|
||||
- name: Upload dependency validation reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: dependency-maintenance-results
|
||||
path: |
|
||||
python/scripts/dependencies/dependency-bounds-test-results.json
|
||||
python/scripts/dependencies/dependency-range-results.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Create issue for failed dependency bounds test
|
||||
if: steps.validate_bounds_test.outcome != 'success'
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const fs = require("fs")
|
||||
const reportPath = "python/scripts/dependencies/dependency-bounds-test-results.json"
|
||||
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const openIssueTitles = new Set(
|
||||
openIssues.filter((issue) => !issue.pull_request).map((issue) => issue.title)
|
||||
)
|
||||
|
||||
const formatError = (message) => String(message || "No error output captured.").replace(/```/g, "'''")
|
||||
const title = "Dependency bounds test failed"
|
||||
if (openIssueTitles.has(title)) {
|
||||
core.info(`Issue already exists: ${title}`)
|
||||
return
|
||||
}
|
||||
|
||||
const bodyLines = [
|
||||
"Automated dependency bounds test mode failed before dependency upper-bound validation could run.",
|
||||
"",
|
||||
"The weekly dependency maintenance workflow kept only dev dependency updates for the generated PR, if any, and skipped dependency range updates for this run.",
|
||||
"",
|
||||
]
|
||||
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
|
||||
const failedScenarios = (report.scenarios ?? []).filter((scenario) => scenario.status === "failed")
|
||||
for (const scenario of failedScenarios) {
|
||||
bodyLines.push(`### ${scenario.name} scenario (${scenario.resolution})`)
|
||||
const failedPackages = (scenario.packages ?? []).filter((pkg) => pkg.status === "failed")
|
||||
for (const pkg of failedPackages.slice(0, 10)) {
|
||||
bodyLines.push(
|
||||
"",
|
||||
`- Package: \`${pkg.package_name}\``,
|
||||
`- Project path: \`${pkg.project_path}\``,
|
||||
"",
|
||||
"```",
|
||||
formatError(pkg.error).slice(0, 3500),
|
||||
"```"
|
||||
)
|
||||
}
|
||||
if (failedPackages.length > 10) {
|
||||
bodyLines.push("", `_Additional failed packages omitted: ${failedPackages.length - 10}_`)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
bodyLines.push(`No dependency bounds test report was found at \`${reportPath}\`.`)
|
||||
}
|
||||
|
||||
bodyLines.push("", `Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`)
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title,
|
||||
body: bodyLines.join("\n"),
|
||||
})
|
||||
core.info(`Created issue: ${title}`)
|
||||
|
||||
- name: Create issues for failed dependency candidates
|
||||
if: always()
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const fs = require("fs")
|
||||
const reportPath = "python/scripts/dependencies/dependency-range-results.json"
|
||||
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
core.info(`No dependency range report found at ${reportPath}`)
|
||||
return
|
||||
}
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
|
||||
const dependencyFailures = []
|
||||
|
||||
for (const packageResult of report.packages ?? []) {
|
||||
for (const dependency of packageResult.dependencies ?? []) {
|
||||
const candidateVersions = new Set(dependency.candidate_versions ?? [])
|
||||
const failedAttempts = (dependency.attempts ?? []).filter(
|
||||
(attempt) => attempt.status === "failed" && candidateVersions.has(attempt.trial_upper)
|
||||
)
|
||||
if (!failedAttempts.length) {
|
||||
continue
|
||||
}
|
||||
|
||||
const failuresByVersion = new Map()
|
||||
for (const attempt of failedAttempts) {
|
||||
const version = attempt.trial_upper || "unknown"
|
||||
if (!failuresByVersion.has(version)) {
|
||||
failuresByVersion.set(version, attempt.error || "No error output captured.")
|
||||
}
|
||||
}
|
||||
|
||||
dependencyFailures.push({
|
||||
packageName: packageResult.package_name,
|
||||
projectPath: packageResult.project_path,
|
||||
dependencyName: dependency.name,
|
||||
originalRequirements: dependency.original_requirements ?? [],
|
||||
finalRequirements: dependency.final_requirements ?? [],
|
||||
failedVersions: [...failuresByVersion.entries()].map(([version, error]) => ({ version, error })),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!dependencyFailures.length) {
|
||||
core.info("No failing dependency candidates found.")
|
||||
return
|
||||
}
|
||||
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const openIssueTitles = new Set(
|
||||
openIssues.filter((issue) => !issue.pull_request).map((issue) => issue.title)
|
||||
)
|
||||
|
||||
const formatError = (message) => String(message || "No error output captured.").replace(/```/g, "'''")
|
||||
|
||||
for (const failure of dependencyFailures) {
|
||||
const title = `Dependency validation failed: ${failure.dependencyName} (${failure.packageName})`
|
||||
if (openIssueTitles.has(title)) {
|
||||
core.info(`Issue already exists: ${title}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const visibleFailures = failure.failedVersions.slice(0, 5)
|
||||
const omittedCount = failure.failedVersions.length - visibleFailures.length
|
||||
const failureDetails = visibleFailures
|
||||
.map(
|
||||
(entry) =>
|
||||
`- \`${entry.version}\`\n\n\`\`\`\n${formatError(entry.error).slice(0, 3500)}\n\`\`\``
|
||||
)
|
||||
.join("\n\n")
|
||||
|
||||
const body = [
|
||||
"Automated dependency range validation found candidate versions that failed checks.",
|
||||
"",
|
||||
`- Package: \`${failure.packageName}\``,
|
||||
`- Project path: \`${failure.projectPath}\``,
|
||||
`- Dependency: \`${failure.dependencyName}\``,
|
||||
`- Original requirements: ${
|
||||
failure.originalRequirements.length
|
||||
? failure.originalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
`- Final requirements after run: ${
|
||||
failure.finalRequirements.length
|
||||
? failure.finalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
"",
|
||||
"### Failed versions and errors",
|
||||
failureDetails,
|
||||
omittedCount > 0 ? `\n_Additional failed versions omitted: ${omittedCount}_` : "",
|
||||
"",
|
||||
`Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
|
||||
].join("\n")
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title,
|
||||
body,
|
||||
})
|
||||
openIssueTitles.add(title)
|
||||
core.info(`Created issue: ${title}`)
|
||||
}
|
||||
|
||||
- name: Keep only dev updates when range validation fails
|
||||
if: steps.validate_bounds_test.outcome != 'success' || steps.validate_ranges.outcome != 'success'
|
||||
env:
|
||||
DEV_PATCH: ${{ steps.dev_changes.outputs.patch }}
|
||||
HAS_DEV_CHANGES: ${{ steps.dev_changes.outputs.has_dev_changes }}
|
||||
run: |
|
||||
git restore python/pyproject.toml python/packages/*/pyproject.toml python/uv.lock
|
||||
if [ "${HAS_DEV_CHANGES}" = "true" ]; then
|
||||
git apply "${DEV_PATCH}"
|
||||
fi
|
||||
|
||||
- name: Refresh lockfile after dependency range updates
|
||||
if: steps.validate_bounds_test.outcome == 'success' && steps.validate_ranges.outcome == 'success'
|
||||
run: uv lock
|
||||
working-directory: ./python
|
||||
|
||||
- name: Install final dependency set
|
||||
run: uv run poe install
|
||||
working-directory: ./python
|
||||
|
||||
- name: Run final checks
|
||||
run: uv run poe check
|
||||
working-directory: ./python
|
||||
|
||||
- name: Run final typing
|
||||
run: uv run poe typing
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dependency updates
|
||||
id: commit_updates
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-maintenance"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/pyproject.toml python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "Python: chore: update dependencies"
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update dependency maintenance tracking issue
|
||||
if: steps.commit_updates.outputs.has_changes == 'true'
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const branch = "automation/python-dependency-maintenance"
|
||||
const prTitle = "Python: chore: update dependencies"
|
||||
const issueTitle = "Python dependency maintenance PR needed"
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
|
||||
const branchRef = await github.rest.git.getRef({
|
||||
owner,
|
||||
repo,
|
||||
ref: `heads/${branch}`,
|
||||
})
|
||||
const branchSha = branchRef.data.object.sha
|
||||
const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`
|
||||
const compareUrl = `${context.serverUrl}/${owner}/${repo}/compare/main...${branch}`
|
||||
|
||||
const prBody = [
|
||||
"### Motivation & Context",
|
||||
"",
|
||||
"This automated update keeps Python dependency metadata coherent across the uv workspace. Python dependencies can be declared in multiple `pyproject.toml` files, but the workspace has one shared `python/uv.lock`, so dependency maintenance should update and validate them together instead of through per-manifest Dependabot PRs.",
|
||||
"",
|
||||
"### Description & Review Guide",
|
||||
"",
|
||||
"- **What are the major changes?** Refresh Python dev dependency pins, update package dependency ranges when the bounds tooling succeeds, and refresh `python/uv.lock`.",
|
||||
"- **What is the impact of these changes?** Keeps the Python workspace dependency set current while producing at most one dependency PR for the week. If dependency range validation fails, this PR contains only the dev dependency updates that still pass final validation, and separate issues track failed range candidates.",
|
||||
"- **What do you want reviewers to focus on?** Review the generated dependency metadata changes and any dependency-range updates for package-specific compatibility concerns.",
|
||||
'<!-- NOTE FOR AUTOMATED/AI REVIEWERS: the "What do you want reviewers to focus on?"',
|
||||
" item above is intended for human reviewers only. Automated/AI reviewers should",
|
||||
" ignore it and review the entire change rather than narrowing scope to it. -->",
|
||||
"",
|
||||
"",
|
||||
"### Related Issue",
|
||||
"",
|
||||
"No linked issue; this PR is generated by scheduled Python dependency maintenance.",
|
||||
"",
|
||||
"### Contribution Checklist",
|
||||
"",
|
||||
"- [x] The code builds clean without any errors or warnings",
|
||||
"- [x] All unit tests pass, and I have added new tests where possible",
|
||||
"- [x] The PR follows the [Contribution Guidelines](https://github.com/microsoft/agent-framework/blob/main/CONTRIBUTING.md)",
|
||||
"- [ ] This PR is linked to an issue and there is no other open PR for this issue (see Related Issue above).",
|
||||
'- [x] **This is not a breaking change.** If it _is_ a breaking change, add the `breaking change` label (or add "[BREAKING]" to the title prefix, before or after any language prefix) — a workflow keeps the label and title prefix in sync automatically.',
|
||||
].join("\n")
|
||||
|
||||
const prBodyFence = "```"
|
||||
const command = [
|
||||
"PR_BODY_FILE=\"$(mktemp)\"",
|
||||
`cat > "$PR_BODY_FILE" <<'EOF'`,
|
||||
prBody,
|
||||
"EOF",
|
||||
"gh pr create --repo microsoft/agent-framework --base main \\",
|
||||
` --head ${owner}:${branch} \\`,
|
||||
` --title "${prTitle}" \\`,
|
||||
" --body-file \"$PR_BODY_FILE\"",
|
||||
].join("\n")
|
||||
|
||||
const issueBody = [
|
||||
"The Python dependency maintenance workflow generated and validated dependency updates, then pushed them to the automation branch.",
|
||||
"",
|
||||
`- Branch: \`${branch}\``,
|
||||
`- Commit: \`${branchSha}\``,
|
||||
`- Compare: ${compareUrl}`,
|
||||
`- Workflow run: ${runUrl}`,
|
||||
"",
|
||||
"GitHub Actions is not permitted to create pull requests in this repository, so a maintainer needs to create the PR manually.",
|
||||
"",
|
||||
"### Create the PR",
|
||||
"",
|
||||
"```bash",
|
||||
command,
|
||||
"```",
|
||||
"",
|
||||
"### Generated PR body",
|
||||
"",
|
||||
prBodyFence,
|
||||
prBody,
|
||||
prBodyFence,
|
||||
].join("\n")
|
||||
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const existingIssue = openIssues.find((issue) => !issue.pull_request && issue.title === issueTitle)
|
||||
|
||||
if (existingIssue) {
|
||||
await github.rest.issues.update({
|
||||
owner,
|
||||
repo,
|
||||
issue_number: existingIssue.number,
|
||||
title: issueTitle,
|
||||
body: issueBody,
|
||||
})
|
||||
core.info(`Updated issue #${existingIssue.number}: ${issueTitle}`)
|
||||
} else {
|
||||
const createdIssue = await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title: issueTitle,
|
||||
body: issueBody,
|
||||
})
|
||||
core.info(`Created issue #${createdIssue.data.number}: ${issueTitle}`)
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
# Probe the highest allowed dependency versions, then open issues/PRs from the passing updates.
|
||||
name: Python - Dependency Range Validation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
dependency-range-validation:
|
||||
name: Dependency Range Validation
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# For now only run 3.13, if we do encounter situations where there are mismatches between packages and python versions (other then 3.10 and 3.14 which are known to not be able to install everything)
|
||||
# then we will have to reevaluate.
|
||||
UV_PYTHON: "3.13"
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Run dependency range validation
|
||||
id: validate_ranges
|
||||
# Keep workflow running so we can still publish diagnostics from this run.
|
||||
continue-on-error: true
|
||||
run: uv run poe validate-dependency-bounds-project --mode upper --package "*"
|
||||
working-directory: ./python
|
||||
|
||||
- name: Upload dependency range report
|
||||
# Always publish the report so failures are inspectable even when validation fails.
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dependency-range-results
|
||||
path: python/scripts/dependencies/dependency-range-results.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Create issues for failed dependency candidates
|
||||
# Always process the report so failed candidates create actionable tracking issues.
|
||||
if: always()
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const fs = require("fs")
|
||||
const reportPath = "python/scripts/dependencies/dependency-range-results.json"
|
||||
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
core.warning(`No dependency range report found at ${reportPath}`)
|
||||
return
|
||||
}
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
|
||||
const dependencyFailures = []
|
||||
|
||||
for (const packageResult of report.packages ?? []) {
|
||||
for (const dependency of packageResult.dependencies ?? []) {
|
||||
const candidateVersions = new Set(dependency.candidate_versions ?? [])
|
||||
const failedAttempts = (dependency.attempts ?? []).filter(
|
||||
(attempt) => attempt.status === "failed" && candidateVersions.has(attempt.trial_upper)
|
||||
)
|
||||
if (!failedAttempts.length) {
|
||||
continue
|
||||
}
|
||||
|
||||
const failuresByVersion = new Map()
|
||||
for (const attempt of failedAttempts) {
|
||||
const version = attempt.trial_upper || "unknown"
|
||||
if (!failuresByVersion.has(version)) {
|
||||
failuresByVersion.set(version, attempt.error || "No error output captured.")
|
||||
}
|
||||
}
|
||||
|
||||
dependencyFailures.push({
|
||||
packageName: packageResult.package_name,
|
||||
projectPath: packageResult.project_path,
|
||||
dependencyName: dependency.name,
|
||||
originalRequirements: dependency.original_requirements ?? [],
|
||||
finalRequirements: dependency.final_requirements ?? [],
|
||||
failedVersions: [...failuresByVersion.entries()].map(([version, error]) => ({ version, error })),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!dependencyFailures.length) {
|
||||
core.info("No failing dependency candidates found.")
|
||||
return
|
||||
}
|
||||
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const openIssueTitles = new Set(
|
||||
openIssues.filter((issue) => !issue.pull_request).map((issue) => issue.title)
|
||||
)
|
||||
|
||||
const formatError = (message) => String(message || "No error output captured.").replace(/```/g, "'''")
|
||||
|
||||
for (const failure of dependencyFailures) {
|
||||
const title = `Dependency validation failed: ${failure.dependencyName} (${failure.packageName})`
|
||||
if (openIssueTitles.has(title)) {
|
||||
core.info(`Issue already exists: ${title}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const visibleFailures = failure.failedVersions.slice(0, 5)
|
||||
const omittedCount = failure.failedVersions.length - visibleFailures.length
|
||||
const failureDetails = visibleFailures
|
||||
.map(
|
||||
(entry) =>
|
||||
`- \`${entry.version}\`\n\n\`\`\`\n${formatError(entry.error).slice(0, 3500)}\n\`\`\``
|
||||
)
|
||||
.join("\n\n")
|
||||
|
||||
const body = [
|
||||
"Automated dependency range validation found candidate versions that failed checks.",
|
||||
"",
|
||||
`- Package: \`${failure.packageName}\``,
|
||||
`- Project path: \`${failure.projectPath}\``,
|
||||
`- Dependency: \`${failure.dependencyName}\``,
|
||||
`- Original requirements: ${
|
||||
failure.originalRequirements.length
|
||||
? failure.originalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
`- Final requirements after run: ${
|
||||
failure.finalRequirements.length
|
||||
? failure.finalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
"",
|
||||
"### Failed versions and errors",
|
||||
failureDetails,
|
||||
omittedCount > 0 ? `\n_Additional failed versions omitted: ${omittedCount}_` : "",
|
||||
"",
|
||||
`Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
|
||||
].join("\n")
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title,
|
||||
body,
|
||||
})
|
||||
openIssueTitles.add(title)
|
||||
core.info(`Created issue: ${title}`)
|
||||
}
|
||||
|
||||
- name: Refresh lockfile
|
||||
# Only refresh lockfile after a clean validation to avoid committing known-bad ranges.
|
||||
if: steps.validate_ranges.outcome == 'success'
|
||||
run: uv lock --upgrade
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dependency updates
|
||||
id: commit_updates
|
||||
if: steps.validate_ranges.outcome == 'success'
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-range-updates"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "chore: update dependency ranges"
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update pull request with GitHub CLI
|
||||
# Only open/update PRs for validated updates to keep automation branches trustworthy.
|
||||
if: steps.validate_ranges.outcome == 'success' && steps.commit_updates.outputs.has_changes == 'true'
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-range-updates"
|
||||
PR_TITLE="Python: chore: update dependency ranges"
|
||||
PR_BODY_FILE="$(mktemp)"
|
||||
|
||||
cat > "${PR_BODY_FILE}" <<'EOF'
|
||||
This PR was generated by the dependency range validation workflow.
|
||||
|
||||
- Ran `uv run poe validate-dependency-bounds-project --mode upper --package "*"`
|
||||
- Updated package dependency bounds
|
||||
- Refreshed `python/uv.lock` with `uv lock --upgrade`
|
||||
EOF
|
||||
|
||||
PR_NUMBER="$(gh pr list --head "${BRANCH}" --base main --state open --json number --jq '.[0].number')"
|
||||
if [ -n "${PR_NUMBER}" ]; then
|
||||
gh pr edit "${PR_NUMBER}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
else
|
||||
gh pr create --base main --head "${BRANCH}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
fi
|
||||
@@ -0,0 +1,91 @@
|
||||
name: Python - Dev Dependency Upgrade
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
upgrade-dev-dependencies:
|
||||
name: Upgrade Dev Dependencies
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
UV_PYTHON: "3.13"
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Upgrade dev dependencies and validate workspace
|
||||
run: uv run poe upgrade-dev-dependencies
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dev dependency updates
|
||||
id: commit_updates
|
||||
run: |
|
||||
BRANCH="automation/python-dev-dependency-updates"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/pyproject.toml python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dev dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -F- <<'EOF'
|
||||
Python: chore: upgrade dev dependencies
|
||||
EOF
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update pull request with GitHub CLI
|
||||
if: steps.commit_updates.outputs.has_changes == 'true'
|
||||
run: |
|
||||
BRANCH="automation/python-dev-dependency-updates"
|
||||
PR_TITLE="Python: chore: upgrade dev dependencies"
|
||||
PR_BODY_FILE="$(mktemp)"
|
||||
|
||||
cat > "${PR_BODY_FILE}" <<'EOF'
|
||||
### Motivation and Context
|
||||
|
||||
This automated update refreshes Python dev dependency pins across the workspace and reruns the repo validation gates before opening a pull request.
|
||||
|
||||
### Description
|
||||
|
||||
- Ran `uv run poe upgrade-dev-dependencies`
|
||||
- Refreshed dev dependency pins in workspace `pyproject.toml` files
|
||||
- Refreshed `python/uv.lock` with `uv lock --upgrade`
|
||||
- Reinstalled from the frozen lockfile and reran `check`, `typing`, and `test`
|
||||
|
||||
### Contribution Checklist
|
||||
|
||||
- [x] The code builds clean without any errors or warnings
|
||||
- [x] The PR follows the [Contribution Guidelines](https://github.com/microsoft/agent-framework/blob/main/CONTRIBUTING.md)
|
||||
- [x] All unit tests pass, and I have added new tests where possible
|
||||
- [ ] **Is this a breaking change?** If yes, add "[BREAKING]" prefix to the title of the PR.
|
||||
EOF
|
||||
|
||||
PR_NUMBER="$(gh pr list --head "${BRANCH}" --base main --state open --json number --jq '.[0].number')"
|
||||
if [ -n "${PR_NUMBER}" ]; then
|
||||
gh pr edit "${PR_NUMBER}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
else
|
||||
gh pr create --base main --head "${BRANCH}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
fi
|
||||
@@ -24,9 +24,9 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
version-file: "python/pyproject.toml"
|
||||
enable-cache: true
|
||||
|
||||
@@ -13,25 +13,13 @@ on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
checkout-ref:
|
||||
description: "Immutable commit SHA to check out"
|
||||
description: "Git ref to checkout (e.g., refs/pull/123/head)"
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
ANTHROPIC_API_KEY:
|
||||
required: true
|
||||
AZURE_CLIENT_ID:
|
||||
required: true
|
||||
AZURE_TENANT_ID:
|
||||
required: true
|
||||
AZURE_SUBSCRIPTION_ID:
|
||||
required: true
|
||||
FOUNDRY_MODELS_API_KEY:
|
||||
required: false
|
||||
OPENAI__APIKEY:
|
||||
required: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
@@ -48,7 +36,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
@@ -72,8 +60,8 @@ jobs:
|
||||
environment: integration
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OPENAI_RESPONSES_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
|
||||
@@ -81,7 +69,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
@@ -99,35 +87,24 @@ jobs:
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-openai
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Azure OpenAI integration tests
|
||||
python-tests-azure-openai:
|
||||
name: Python Integration Tests - Azure OpenAI
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
@@ -138,7 +115,7 @@ jobs:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
@@ -149,20 +126,14 @@ jobs:
|
||||
packages/openai/tests/openai/test_openai_chat_completion_client_azure.py
|
||||
packages/openai/tests/openai/test_openai_chat_client_azure.py
|
||||
packages/openai/tests/openai/test_openai_embedding_client_azure.py
|
||||
packages/azure-ai/tests/azure_openai
|
||||
--ignore=packages/azure-ai/tests/azure_openai/test_azure_responses_client_foundry.py
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-azure-openai
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Misc integration tests (Anthropic, Hyperlight, Ollama, MCP)
|
||||
# Misc integration tests (Anthropic, Ollama, MCP)
|
||||
python-tests-misc-integration:
|
||||
name: Python Integration Tests - Misc
|
||||
runs-on: ubuntu-latest
|
||||
@@ -170,15 +141,13 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
|
||||
ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
|
||||
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
|
||||
OLLAMA_MODEL: qwen2.5:1.5b
|
||||
OLLAMA_EMBEDDING_MODEL: nomic-embed-text
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
@@ -188,43 +157,6 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Install Ollama
|
||||
run: curl -fsSL https://ollama.com/install.sh | sh
|
||||
working-directory: .
|
||||
- name: Cache Ollama models
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.ollama/models
|
||||
key: ollama-models-qwen2.5-1.5b-nomic-embed-text-v1
|
||||
- name: Start Ollama and pull models
|
||||
run: |
|
||||
# Stop any Ollama instance auto-started by the install script
|
||||
pkill ollama || true
|
||||
sleep 2
|
||||
ollama serve &
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# Pull models with retry for transient 429 rate limits
|
||||
for model in qwen2.5:1.5b nomic-embed-text; do
|
||||
pulled=false
|
||||
for attempt in 1 2 3; do
|
||||
if ollama pull "$model"; then
|
||||
pulled=true
|
||||
break
|
||||
fi
|
||||
echo "Retry $attempt for $model (waiting 15s)..."
|
||||
sleep 15
|
||||
done
|
||||
if [ "$pulled" != "true" ]; then
|
||||
echo "ERROR: Failed to pull $model after 3 attempts"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
working-directory: .
|
||||
- name: Start local MCP server
|
||||
id: local-mcp
|
||||
uses: ./.github/actions/setup-local-mcp-server
|
||||
@@ -232,27 +164,16 @@ jobs:
|
||||
fallback_url: ${{ env.LOCAL_MCP_URL }}
|
||||
- name: Prefer local MCP URL when available
|
||||
run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV"
|
||||
- name: Test with pytest (Anthropic, Hyperlight, Mistral, Ollama, MCP integration)
|
||||
- name: Test with pytest (Anthropic, Ollama, MCP integration)
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/anthropic/tests
|
||||
packages/hyperlight/tests
|
||||
packages/mistral/tests
|
||||
packages/ollama/tests
|
||||
packages/core/tests/core/test_mcp.py
|
||||
packages/hosting-mcp/tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 30
|
||||
--junitxml=pytest.xml
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-misc
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
--retries 2 --retry-delay 5
|
||||
- name: Stop local MCP server
|
||||
if: always()
|
||||
shell: bash
|
||||
@@ -274,30 +195,32 @@ jobs:
|
||||
done
|
||||
kill -KILL -- "-$server_pid" 2>/dev/null || kill -KILL "$server_pid" 2>/dev/null || true
|
||||
|
||||
# Foundry integration tests
|
||||
python-tests-foundry:
|
||||
name: Python Integration Tests - Foundry
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
# Azure Functions + Durable Task integration tests
|
||||
python-tests-functions:
|
||||
name: Python Integration Tests - Functions
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
UV_PYTHON: "3.11"
|
||||
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OPENAI_RESPONSES_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
|
||||
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME }}
|
||||
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION }}
|
||||
FOUNDRY_MODELS_ENDPOINT: ${{ vars.FOUNDRY_MODELS_ENDPOINT || '' }}
|
||||
FOUNDRY_MODELS_API_KEY: ${{ secrets.FOUNDRY_MODELS_API_KEY || '' }}
|
||||
FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }}
|
||||
FOUNDRY_IMAGE_EMBEDDING_MODEL: ${{ vars.FOUNDRY_IMAGE_EMBEDDING_MODEL || '' }}
|
||||
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
|
||||
FUNCTIONS_WORKER_RUNTIME: "python"
|
||||
DURABLE_TASK_SCHEDULER_CONNECTION_STRING: "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"
|
||||
AzureWebJobsStorage: "UseDevelopmentStorage=true"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
@@ -308,7 +231,55 @@ jobs:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
- name: Set up Azure Functions Integration Test Emulators
|
||||
uses: ./.github/actions/azure-functions-integration-setup
|
||||
id: azure-functions-setup
|
||||
- name: Test with pytest (Functions + Durable Task integration)
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/azurefunctions/tests/integration_tests
|
||||
packages/durabletask/tests/integration_tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
-x
|
||||
--timeout=360 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
|
||||
# Foundry integration tests
|
||||
python-tests-foundry:
|
||||
name: Python Integration Tests - Foundry
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }}
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREAI__DEPLOYMENTNAME }}
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME }}
|
||||
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION }}
|
||||
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
@@ -317,69 +288,12 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/azure-ai/tests/azure_openai/test_azure_responses_client_foundry.py
|
||||
packages/foundry/tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-foundry
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Foundry Hosting integration tests
|
||||
python-tests-foundry-hosting:
|
||||
name: Python Integration Tests - Foundry Hosting
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
- name: Test with pytest (Foundry Hosting integration)
|
||||
timeout-minutes: 15
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/foundry_hosting/tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-foundry-hosting
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Azure Cosmos integration tests
|
||||
python-tests-cosmos:
|
||||
@@ -402,7 +316,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
@@ -424,123 +338,7 @@ jobs:
|
||||
echo "Cosmos DB emulator did not become ready in time." >&2
|
||||
exit 1
|
||||
- name: Test with pytest (Cosmos integration)
|
||||
run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5 --junitxml=${{ github.workspace }}/python/pytest.xml
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-cosmos
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# GitHub Copilot integration tests
|
||||
python-tests-github-copilot:
|
||||
name: Python Integration Tests - GitHub Copilot
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
permissions:
|
||||
copilot-requests: write
|
||||
contents: read
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
GITHUB_COPILOT_TIMEOUT: "120"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Test with pytest (GitHub Copilot integration)
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/github_copilot/tests
|
||||
-m integration
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-github-copilot
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Integration test trend report (aggregates per-job JUnit XML results)
|
||||
python-integration-test-report:
|
||||
name: Integration Test Report
|
||||
if: >
|
||||
always() &&
|
||||
(contains(join(needs.*.result, ','), 'success') ||
|
||||
contains(join(needs.*.result, ','), 'failure'))
|
||||
needs:
|
||||
[
|
||||
python-tests-openai,
|
||||
python-tests-azure-openai,
|
||||
python-tests-misc-integration,
|
||||
python-tests-foundry,
|
||||
python-tests-foundry-hosting,
|
||||
python-tests-cosmos,
|
||||
python-tests-github-copilot,
|
||||
]
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ inputs.checkout-ref }}
|
||||
persist-credentials: false
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Download all test results from current run
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: test-results-*
|
||||
path: test-results/
|
||||
- name: Restore report history cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: python/integration-report-history.json
|
||||
key: integration-report-history-integration-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
integration-report-history-integration-
|
||||
- name: Generate trend report
|
||||
run: >
|
||||
uv run python scripts/integration_test_report/aggregate.py
|
||||
../test-results/
|
||||
integration-report-history.json
|
||||
integration-test-report.md
|
||||
- name: Post to Job Summary
|
||||
if: always()
|
||||
run: cat integration-test-report.md >> $GITHUB_STEP_SUMMARY
|
||||
- name: Save report history cache
|
||||
if: always()
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: python/integration-report-history.json
|
||||
key: integration-report-history-integration-${{ github.run_id }}
|
||||
- name: Upload unified trend report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: integration-test-report
|
||||
path: |
|
||||
python/integration-test-report.md
|
||||
python/integration-report-history.json
|
||||
run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5
|
||||
|
||||
python-integration-tests-check:
|
||||
if: always()
|
||||
@@ -551,20 +349,19 @@ jobs:
|
||||
python-tests-openai,
|
||||
python-tests-azure-openai,
|
||||
python-tests-misc-integration,
|
||||
python-tests-functions,
|
||||
python-tests-foundry,
|
||||
python-tests-foundry-hosting,
|
||||
python-tests-cosmos,
|
||||
python-tests-github-copilot
|
||||
python-tests-cosmos
|
||||
]
|
||||
steps:
|
||||
- name: Fail workflow if tests failed
|
||||
if: contains(join(needs.*.result, ','), 'failure')
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: core.setFailed('Integration Tests Failed!')
|
||||
|
||||
- name: Fail workflow if tests cancelled
|
||||
if: contains(join(needs.*.result, ','), 'cancelled')
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: core.setFailed('Integration Tests Cancelled!')
|
||||
|
||||
@@ -24,17 +24,13 @@ jobs:
|
||||
outputs:
|
||||
pythonChanges: ${{ steps.filter.outputs.python}}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
python:
|
||||
- 'python/**'
|
||||
- '!python/AGENTS.md'
|
||||
- '!python/**/AGENTS.md'
|
||||
- '!python/.github/skills/*'
|
||||
- '!python/.github/skills/**'
|
||||
# run only if 'python' files were changed
|
||||
- name: python tests
|
||||
if: steps.filter.outputs.python == 'true'
|
||||
@@ -63,7 +59,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
@@ -71,7 +67,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
os: ${{ runner.os }}
|
||||
exclude-packages: ${{ matrix.python-version == '3.10' && 'agent-framework-github-copilot agent-framework-azure-cosmos-memory' || '' }}
|
||||
exclude-packages: ${{ matrix.python-version == '3.10' && 'agent-framework-github-copilot' || '' }}
|
||||
env:
|
||||
# Configure a constant location for the uv cache
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
@@ -92,10 +88,13 @@ jobs:
|
||||
- name: Run lab type checking
|
||||
run: cd packages/lab && uv run poe pyright
|
||||
|
||||
- name: Run lab mypy
|
||||
run: cd packages/lab && uv run poe mypy
|
||||
|
||||
# Surface failing tests
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/packages/lab/**.xml
|
||||
summary: true
|
||||
|
||||
@@ -36,13 +36,12 @@ jobs:
|
||||
openaiChanged: ${{ steps.filter.outputs.openai }}
|
||||
azureChanged: ${{ steps.filter.outputs.azure }}
|
||||
miscChanged: ${{ steps.filter.outputs.misc }}
|
||||
foundryChanged: ${{ steps.filter.outputs.foundry }}
|
||||
foundryHostingChanged: ${{ steps.filter.outputs.foundry_hosting }}
|
||||
functionsChanged: ${{ steps.filter.outputs.functions }}
|
||||
azureAiChanged: ${{ steps.filter.outputs.azure-ai }}
|
||||
cosmosChanged: ${{ steps.filter.outputs.cosmos }}
|
||||
githubCopilotChanged: ${{ steps.filter.outputs.github_copilot }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
@@ -63,29 +62,27 @@ jobs:
|
||||
azure:
|
||||
- 'python/packages/openai/**'
|
||||
- 'python/packages/core/agent_framework/azure/**'
|
||||
- 'python/samples/**/providers/azure/**'
|
||||
- 'python/packages/azure-ai/agent_framework_azure_ai/_deprecated_azure_openai.py'
|
||||
- 'python/packages/azure-ai/tests/azure_openai/**'
|
||||
- 'python/samples/**/providers/azure/openai_chat_completion_client_azure*.py'
|
||||
misc:
|
||||
- 'python/packages/anthropic/**'
|
||||
- 'python/packages/hyperlight/**'
|
||||
- 'python/packages/mistral/**'
|
||||
- 'python/packages/ollama/**'
|
||||
- 'python/packages/core/agent_framework/_mcp.py'
|
||||
- 'python/packages/core/tests/core/test_mcp.py'
|
||||
- 'python/packages/hosting-mcp/**'
|
||||
- 'python/scripts/local_mcp_streamable_http_server.py'
|
||||
- '.github/actions/setup-local-mcp-server/**'
|
||||
- '.github/workflows/python-merge-tests.yml'
|
||||
- '.github/workflows/python-integration-tests.yml'
|
||||
foundry:
|
||||
functions:
|
||||
- 'python/packages/azurefunctions/**'
|
||||
- 'python/packages/durabletask/**'
|
||||
azure-ai:
|
||||
- 'python/packages/azure-ai/**'
|
||||
- 'python/packages/foundry/**'
|
||||
- 'python/samples/**/providers/foundry/**'
|
||||
- 'python/samples/02-agents/embeddings/foundry_embeddings.py'
|
||||
foundry_hosting:
|
||||
- 'python/packages/foundry_hosting/**'
|
||||
cosmos:
|
||||
- 'python/packages/azure-cosmos/**'
|
||||
github_copilot:
|
||||
- 'python/packages/github_copilot/**'
|
||||
# run only if 'python' files were changed
|
||||
- name: python tests
|
||||
if: steps.filter.outputs.python == 'true'
|
||||
@@ -107,7 +104,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
@@ -120,13 +117,12 @@ jobs:
|
||||
-m "not integration"
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
working-directory: ./python
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
@@ -145,8 +141,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
env:
|
||||
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OPENAI_RESPONSES_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
|
||||
@@ -154,7 +150,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
@@ -169,7 +165,6 @@ jobs:
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
working-directory: ./python
|
||||
- name: Test OpenAI samples
|
||||
timeout-minutes: 10
|
||||
@@ -178,20 +173,13 @@ jobs:
|
||||
working-directory: ./python
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: OpenAI integration test results
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-openai
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Azure OpenAI integration tests
|
||||
python-tests-azure-openai:
|
||||
@@ -206,16 +194,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
env:
|
||||
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
@@ -224,7 +212,7 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
@@ -235,11 +223,12 @@ jobs:
|
||||
packages/openai/tests/openai/test_openai_chat_completion_client_azure.py
|
||||
packages/openai/tests/openai/test_openai_chat_client_azure.py
|
||||
packages/openai/tests/openai/test_openai_embedding_client_azure.py
|
||||
packages/azure-ai/tests/azure_openai
|
||||
--ignore=packages/azure-ai/tests/azure_openai/test_azure_responses_client_foundry.py
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
working-directory: ./python
|
||||
- name: Test Azure samples
|
||||
timeout-minutes: 10
|
||||
@@ -248,20 +237,13 @@ jobs:
|
||||
working-directory: ./python
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: Azure OpenAI integration test results
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-azure-openai
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Misc integration tests (Anthropic, Ollama, MCP)
|
||||
python-tests-misc-integration:
|
||||
@@ -277,58 +259,19 @@ jobs:
|
||||
environment: integration
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
|
||||
ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
|
||||
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
|
||||
OLLAMA_MODEL: qwen2.5:1.5b
|
||||
OLLAMA_EMBEDDING_MODEL: nomic-embed-text
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Install Ollama
|
||||
run: curl -fsSL https://ollama.com/install.sh | sh
|
||||
working-directory: .
|
||||
- name: Cache Ollama models
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||||
with:
|
||||
path: ~/.ollama/models
|
||||
key: ollama-models-qwen2.5-1.5b-nomic-embed-text-v1
|
||||
- name: Start Ollama and pull models
|
||||
run: |
|
||||
# Stop any Ollama instance auto-started by the install script
|
||||
pkill ollama || true
|
||||
sleep 2
|
||||
ollama serve &
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
# Pull models with retry for transient 429 rate limits
|
||||
for model in qwen2.5:1.5b nomic-embed-text; do
|
||||
pulled=false
|
||||
for attempt in 1 2 3; do
|
||||
if ollama pull "$model"; then
|
||||
pulled=true
|
||||
break
|
||||
fi
|
||||
echo "Retry $attempt for $model (waiting 15s)..."
|
||||
sleep 15
|
||||
done
|
||||
if [ "$pulled" != "true" ]; then
|
||||
echo "ERROR: Failed to pull $model after 3 attempts"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
working-directory: .
|
||||
- name: Start local MCP server
|
||||
id: local-mcp
|
||||
uses: ./.github/actions/setup-local-mcp-server
|
||||
@@ -336,20 +279,16 @@ jobs:
|
||||
fallback_url: ${{ env.LOCAL_MCP_URL }}
|
||||
- name: Prefer local MCP URL when available
|
||||
run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV"
|
||||
- name: Test with pytest (Anthropic, Hyperlight, Mistral, Ollama, MCP integration)
|
||||
- name: Test with pytest (Anthropic, Ollama, MCP integration)
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/anthropic/tests
|
||||
packages/hyperlight/tests
|
||||
packages/mistral/tests
|
||||
packages/ollama/tests
|
||||
packages/core/tests/core/test_mcp.py
|
||||
packages/hosting-mcp/tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 30
|
||||
--junitxml=pytest.xml
|
||||
--retries 2 --retry-delay 5
|
||||
working-directory: ./python
|
||||
- name: Stop local MCP server
|
||||
if: always()
|
||||
@@ -373,20 +312,82 @@ jobs:
|
||||
kill -KILL -- "-$server_pid" 2>/dev/null || kill -KILL "$server_pid" 2>/dev/null || true
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: Misc integration test results
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
|
||||
# Azure Functions + Durable Task integration tests
|
||||
python-tests-functions:
|
||||
name: Python Tests - Functions Integration
|
||||
needs: paths-filter
|
||||
if: >
|
||||
github.event_name != 'pull_request' &&
|
||||
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
||||
(github.event_name != 'merge_group' ||
|
||||
needs.paths-filter.outputs.functionsChanged == 'true' ||
|
||||
needs.paths-filter.outputs.coreChanged == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
env:
|
||||
UV_PYTHON: "3.11"
|
||||
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
||||
OPENAI_RESPONSES_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
||||
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
|
||||
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
||||
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
FUNCTIONS_WORKER_RUNTIME: "python"
|
||||
DURABLE_TASK_SCHEDULER_CONNECTION_STRING: "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"
|
||||
AzureWebJobsStorage: "UseDevelopmentStorage=true"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
name: test-results-misc
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
- name: Set up Azure Functions Integration Test Emulators
|
||||
uses: ./.github/actions/azure-functions-integration-setup
|
||||
id: azure-functions-setup
|
||||
- name: Test with pytest (Functions + Durable Task integration)
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/azurefunctions/tests/integration_tests
|
||||
packages/durabletask/tests/integration_tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
-x
|
||||
--timeout=360 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
working-directory: ./python
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: Functions integration test results
|
||||
|
||||
python-tests-foundry:
|
||||
name: Python Integration Tests - Foundry
|
||||
@@ -395,25 +396,23 @@ jobs:
|
||||
github.event_name != 'pull_request' &&
|
||||
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
||||
(github.event_name != 'merge_group' ||
|
||||
needs.paths-filter.outputs.foundryChanged == 'true' ||
|
||||
needs.paths-filter.outputs.azureAiChanged == 'true' ||
|
||||
needs.paths-filter.outputs.coreChanged == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
env:
|
||||
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }}
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREAI__DEPLOYMENTNAME }}
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME }}
|
||||
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION }}
|
||||
FOUNDRY_MODELS_ENDPOINT: ${{ vars.FOUNDRY_MODELS_ENDPOINT || '' }}
|
||||
FOUNDRY_MODELS_API_KEY: ${{ secrets.FOUNDRY_MODELS_API_KEY || '' }}
|
||||
FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }}
|
||||
FOUNDRY_IMAGE_EMBEDDING_MODEL: ${{ vars.FOUNDRY_IMAGE_EMBEDDING_MODEL || '' }}
|
||||
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
@@ -422,7 +421,7 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
@@ -431,90 +430,27 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/azure-ai/tests/azure_openai/test_azure_responses_client_foundry.py
|
||||
packages/foundry/tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
working-directory: ./python
|
||||
- name: Test Azure AI samples
|
||||
timeout-minutes: 10
|
||||
if: env.RUN_SAMPLES_TESTS == 'true'
|
||||
run: uv run pytest tests/samples/ -m "azure-ai"
|
||||
working-directory: ./python
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: Test results
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-foundry
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Foundry Hosting integration tests
|
||||
python-tests-foundry-hosting:
|
||||
name: Python Tests - Foundry Hosting Integration
|
||||
needs: paths-filter
|
||||
if: >
|
||||
github.event_name != 'pull_request' &&
|
||||
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
||||
(github.event_name != 'merge_group' ||
|
||||
needs.paths-filter.outputs.foundryHostingChanged == 'true' ||
|
||||
needs.paths-filter.outputs.coreChanged == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
env:
|
||||
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
||||
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Azure CLI Login
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
- name: Test with pytest (Foundry Hosting integration)
|
||||
timeout-minutes: 15
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/foundry_hosting/tests
|
||||
-m integration
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
working-directory: ./python
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: Foundry Hosting integration test results
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-foundry-hosting
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# TODO: Add python-tests-lab
|
||||
|
||||
@@ -545,7 +481,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
@@ -564,143 +500,17 @@ jobs:
|
||||
echo "Cosmos DB emulator did not become ready in time." >&2
|
||||
exit 1
|
||||
- name: Test with pytest (Cosmos integration)
|
||||
run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5 --junitxml=${{ github.workspace }}/python/pytest.xml
|
||||
run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5
|
||||
working-directory: ./python
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: Cosmos integration test results
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-cosmos
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# GitHub Copilot integration tests
|
||||
python-tests-github-copilot:
|
||||
name: Python Tests - GitHub Copilot Integration
|
||||
needs: paths-filter
|
||||
if: >
|
||||
github.event_name != 'pull_request' &&
|
||||
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
||||
(github.event_name != 'merge_group' ||
|
||||
needs.paths-filter.outputs.githubCopilotChanged == 'true' ||
|
||||
needs.paths-filter.outputs.coreChanged == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
permissions:
|
||||
copilot-requests: write
|
||||
contents: read
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
GITHUB_COPILOT_TIMEOUT: "120"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Test with pytest (GitHub Copilot integration)
|
||||
run: >
|
||||
uv run pytest --import-mode=importlib
|
||||
packages/github_copilot/tests
|
||||
-m integration
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
--junitxml=pytest.xml
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
title: GitHub Copilot integration test results
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: test-results-github-copilot
|
||||
path: ./python/pytest.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Integration test trend report (aggregates per-job JUnit XML results)
|
||||
python-integration-test-report:
|
||||
name: Integration Test Report
|
||||
if: >
|
||||
always() &&
|
||||
(contains(join(needs.*.result, ','), 'success') ||
|
||||
contains(join(needs.*.result, ','), 'failure'))
|
||||
needs:
|
||||
[
|
||||
python-tests-openai,
|
||||
python-tests-azure-openai,
|
||||
python-tests-misc-integration,
|
||||
python-tests-foundry,
|
||||
python-tests-foundry-hosting,
|
||||
python-tests-cosmos,
|
||||
python-tests-github-copilot,
|
||||
]
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
- name: Download all test results from current run
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: test-results-*
|
||||
path: test-results/
|
||||
- name: Restore report history cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: python/integration-report-history.json
|
||||
key: integration-report-history-merge-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
integration-report-history-merge-
|
||||
- name: Generate trend report
|
||||
run: >
|
||||
uv run python scripts/integration_test_report/aggregate.py
|
||||
../test-results/
|
||||
integration-report-history.json
|
||||
integration-test-report.md
|
||||
- name: Post to Job Summary
|
||||
if: always()
|
||||
run: cat integration-test-report.md >> $GITHUB_STEP_SUMMARY
|
||||
- name: Save report history cache
|
||||
if: always()
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: python/integration-report-history.json
|
||||
key: integration-report-history-merge-${{ github.run_id }}
|
||||
- name: Upload unified trend report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: integration-test-report
|
||||
path: |
|
||||
python/integration-test-report.md
|
||||
python/integration-report-history.json
|
||||
|
||||
python-integration-tests-check:
|
||||
if: always()
|
||||
@@ -711,22 +521,21 @@ jobs:
|
||||
python-tests-openai,
|
||||
python-tests-azure-openai,
|
||||
python-tests-misc-integration,
|
||||
python-tests-functions,
|
||||
python-tests-foundry,
|
||||
python-tests-foundry-hosting,
|
||||
python-tests-cosmos,
|
||||
python-tests-github-copilot,
|
||||
]
|
||||
steps:
|
||||
- name: Fail workflow if tests failed
|
||||
id: check_tests_failed
|
||||
if: contains(join(needs.*.result, ','), 'failure')
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: core.setFailed('Integration Tests Failed!')
|
||||
|
||||
- name: Fail workflow if tests cancelled
|
||||
id: check_tests_cancelled
|
||||
if: contains(join(needs.*.result, ','), 'cancelled')
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: core.setFailed('Integration Tests Cancelled!')
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
- name: Build the package
|
||||
run: uv run poe --directory packages/${{ env.PACKAGE }} build
|
||||
- name: Release
|
||||
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
python/dist/*
|
||||
|
||||
@@ -8,7 +8,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
@@ -20,46 +19,36 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Download coverage report
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
path: ./python
|
||||
merge-multiple: true
|
||||
- name: Display structure of downloaded files
|
||||
run: ls
|
||||
- name: Read and validate PR number
|
||||
# Keep the artifact handoff aligned with the workflow run that produced it.
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
- name: Read and set PR number
|
||||
# Need to read the PR number from the file saved in the previous workflow
|
||||
# because the workflow_run event does not have access to the PR number
|
||||
# The PR number is needed to post the comment on the PR
|
||||
run: |
|
||||
if [ ! -s pr_number ]; then
|
||||
echo "PR number file 'pr_number' is missing or empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARTIFACT_PR_NUMBER=$(cat pr_number)
|
||||
if ! [[ "$ARTIFACT_PR_NUMBER" =~ ^[0-9]+$ ]]; then
|
||||
echo "::error::PR number file contains invalid content"
|
||||
PR_NUMBER=$(head -1 pr_number | tr -dc '0-9')
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "PR number file 'pr_number' does not contain a valid PR number"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PR_HEAD_SHA=$(gh pr view "$ARTIFACT_PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid')
|
||||
if [ "$PR_HEAD_SHA" != "$RUN_HEAD_SHA" ]; then
|
||||
echo "::error::PR head SHA does not match the triggering workflow run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PR_NUMBER=$ARTIFACT_PR_NUMBER" >> "$GITHUB_ENV"
|
||||
echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV"
|
||||
- name: Pytest coverage comment
|
||||
id: coverageComment
|
||||
uses: MishaKav/pytest-coverage-comment@dd5b80bde6d16941f336518e92929e89069d8451 # v1.7.2
|
||||
uses: MishaKav/pytest-coverage-comment@v1.6.0
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
issue-number: ${{ env.PR_NUMBER }}
|
||||
pytest-xml-coverage-path: python/python-coverage.xml
|
||||
title: "Python Test Coverage Report"
|
||||
|
||||
@@ -6,9 +6,6 @@ on:
|
||||
paths:
|
||||
- "python/packages/**"
|
||||
- "python/tests/unit/**"
|
||||
- "python/scripts/workspace_poe_tasks.py"
|
||||
- ".github/scripts/python_check_coverage.py"
|
||||
- ".github/workflows/python-test-coverage.yml"
|
||||
env:
|
||||
# Configure a constant location for the uv cache
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
@@ -25,7 +22,7 @@ jobs:
|
||||
env:
|
||||
UV_PYTHON: "3.11"
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
# Save the PR number to a file since the workflow_run event
|
||||
# in the coverage report workflow does not have access to it
|
||||
- name: Save PR number
|
||||
@@ -40,12 +37,12 @@ jobs:
|
||||
env:
|
||||
# Configure a constant location for the uv cache
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
- name: Run aggregate tests with coverage report
|
||||
- name: Run all tests with coverage report
|
||||
run: uv run poe test -A -C --cov-report=xml:python-coverage.xml -q --junitxml=pytest.xml
|
||||
- name: Check coverage threshold
|
||||
run: python ${{ github.workspace }}/.github/scripts/python_check_coverage.py python-coverage.xml ${{ env.COVERAGE_THRESHOLD }}
|
||||
run: python ${{ github.workspace }}/.github/workflows/python-check-coverage.py python-coverage.xml ${{ env.COVERAGE_THRESHOLD }}
|
||||
- name: Upload coverage report
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
path: |
|
||||
python/python-coverage.xml
|
||||
|
||||
@@ -5,10 +5,6 @@ on:
|
||||
branches: ["main", "feature*"]
|
||||
paths:
|
||||
- "python/**"
|
||||
- "!python/AGENTS.md"
|
||||
- "!python/**/AGENTS.md"
|
||||
- "!python/.github/skills/*"
|
||||
- "!python/.github/skills/**"
|
||||
env:
|
||||
# Configure a constant location for the uv cache
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
@@ -31,28 +27,28 @@ jobs:
|
||||
run:
|
||||
working-directory: python
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
- name: Set up python and install the project
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
os: ${{ runner.os }}
|
||||
exclude-packages: ${{ matrix.python-version == '3.10' && 'agent-framework-github-copilot agent-framework-azure-cosmos-memory' || '' }}
|
||||
exclude-packages: ${{ matrix.python-version == '3.10' && 'agent-framework-github-copilot' || '' }}
|
||||
env:
|
||||
# Configure a constant location for the uv cache
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
# Unit tests
|
||||
- name: Run all tests
|
||||
run: uv run poe test -A --junitxml=pytest.xml
|
||||
run: uv run poe test -A
|
||||
working-directory: ./python
|
||||
|
||||
# Surface failing tests
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
||||
uses: pmeier/pytest-results-action@v0.7.2
|
||||
with:
|
||||
path: ./python/pytest.xml
|
||||
path: ./python/**.xml
|
||||
summary: true
|
||||
display-options: fEX
|
||||
fail-on-empty: false
|
||||
|
||||
@@ -26,31 +26,14 @@ jobs:
|
||||
ping_stale:
|
||||
name: "Ping stale issues and PRs"
|
||||
runs-on: ubuntu-latest
|
||||
environment: github-app-auth
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Get GitHub automation token
|
||||
id: github-auth
|
||||
uses: ./.github/actions/github-app-token
|
||||
with:
|
||||
mode: ${{ vars.GH_APP_AUTH_MODE }}
|
||||
azure-client-id: ${{ secrets.GH_APP_AZURE_CLIENT_ID }}
|
||||
azure-tenant-id: ${{ secrets.GH_APP_AZURE_TENANT_ID }}
|
||||
azure-subscription-id: ${{ secrets.GH_APP_AZURE_SUBSCRIPTION_ID }}
|
||||
key-vault-name: ${{ secrets.GH_APP_KEY_VAULT_NAME }}
|
||||
key-name: ${{ secrets.GH_APP_KEY_NAME }}
|
||||
github-app-client-id: ${{ secrets.GH_APP_CLIENT_ID }}
|
||||
github-app-installation-id: ${{ secrets.GH_APP_INSTALLATION_ID }}
|
||||
repository: ${{ github.repository }}
|
||||
fallback-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
@@ -60,7 +43,7 @@ jobs:
|
||||
- name: Run stale issue/PR ping
|
||||
run: python .github/scripts/stale_issue_pr_ping.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ steps.github-auth.outputs.token }}
|
||||
GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
TEAM_SLUG: ${{ secrets.DEVELOPER_TEAM }}
|
||||
DAYS_THRESHOLD: ${{ github.event.inputs.days_threshold || '4' }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
|
||||
@@ -47,8 +47,6 @@ htmlcov/
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
pytest.xml
|
||||
python-coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
@@ -136,10 +134,6 @@ celerybeat.pid
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
|
||||
# Foundry agent CLI (contains secrets, auto-generated)
|
||||
.foundry-agent.json
|
||||
.foundry-agent-build.log
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
@@ -206,15 +200,11 @@ temp*/
|
||||
.temp/
|
||||
|
||||
# AI
|
||||
**/.checkpoints/
|
||||
.claude/
|
||||
.omc/
|
||||
.omx/
|
||||
WARP.md
|
||||
**/memory-bank/
|
||||
**/projectBrief.md
|
||||
**/tmpclaude*
|
||||
.kiro/
|
||||
# Dependency-bound validation reports
|
||||
python/scripts/dependency-*-results.json
|
||||
python/scripts/dependencies/dependency-*-results.json
|
||||
@@ -240,14 +230,3 @@ local.settings.json
|
||||
# Database files
|
||||
*.db
|
||||
python/dotnet-ref
|
||||
|
||||
# Generated filtered solution files (created by eng/scripts/New-FilteredSolution.ps1)
|
||||
dotnet/filtered-*.slnx
|
||||
**/*.lscache
|
||||
|
||||
# Local tool state
|
||||
.omc/
|
||||
.omx/
|
||||
|
||||
**/issues/
|
||||
.test_*
|
||||
|
||||
@@ -74,37 +74,6 @@ Contributions must maintain API signature and behavioral compatibility. Contribu
|
||||
that include breaking changes will be rejected. Please file an issue to discuss
|
||||
your idea or change if you believe that a breaking change is warranted.
|
||||
|
||||
#### Automated API Compatibility Validation
|
||||
|
||||
The .NET projects use [Package Validation](https://learn.microsoft.com/dotnet/fundamentals/package-validation/overview)
|
||||
to automatically detect API breaking changes. This validation runs during `dotnet build`
|
||||
(Release configuration) and `dotnet pack`, comparing the current API surface against the
|
||||
latest published NuGet baseline version.
|
||||
|
||||
**What gets validated:** By default, packable RC packages (`IsReleaseCandidate=true`) and
|
||||
GA packages (`IsGenerallyAvailable=true`) that have a published NuGet baseline and do not
|
||||
override validation settings are automatically validated. The shared baseline version and
|
||||
default validation settings are defined in `dotnet/nuget/nuget-package.props`, but
|
||||
individual projects may opt out (for example by setting `EnablePackageValidation=false`).
|
||||
|
||||
**If the build fails with CP errors (e.g., CP0001, CP0002):**
|
||||
|
||||
1. **Unintentional breaking change** — Refactor your code to maintain backward compatibility.
|
||||
2. **Intentional breaking change** (approved by maintainers) — Generate a suppression file:
|
||||
```bash
|
||||
dotnet build <project>.csproj -c Release /p:ApiCompatGenerateSuppressionFile=true
|
||||
```
|
||||
This creates or updates a `CompatibilitySuppressions.xml` in the project directory.
|
||||
Include this file in your PR with justification for the breaking change.
|
||||
|
||||
**After each release:**
|
||||
|
||||
1. Delete all `CompatibilitySuppressions.xml` files from validated projects.
|
||||
2. Update `PackageValidationBaselineVersion` in `dotnet/nuget/nuget-package.props` to the
|
||||
newly published version.
|
||||
|
||||
For more details, see the [Package Validation diagnostic IDs](https://learn.microsoft.com/dotnet/fundamentals/package-validation/diagnostic-ids).
|
||||
|
||||
### Suggested Workflow
|
||||
|
||||
We use and recommend the following workflow:
|
||||
@@ -123,30 +92,22 @@ We use and recommend the following workflow:
|
||||
"issue-123" or "githubhandle-issue".
|
||||
4. Make and commit your changes to your branch.
|
||||
5. Add new tests corresponding to your change, if applicable.
|
||||
6. Run the relevant scripts in [the section below](#development-setup) to ensure that your build is clean and all tests are passing.
|
||||
6. Run the relevant scripts in [the section below](#development-scripts) to ensure that your build is clean and all tests are passing.
|
||||
7. Create a PR against the repository's **main** branch.
|
||||
- State in the description what issue or improvement your change is addressing.
|
||||
- Verify that all the Continuous Integration checks are passing.
|
||||
8. Wait for feedback or approval of your changes from the code maintainers.
|
||||
9. When area owners have signed off, and all checks are green, your PR will be merged.
|
||||
|
||||
### Development Setup
|
||||
### Development scripts
|
||||
|
||||
Each language has its own dev setup guide, coding standards, and build scripts:
|
||||
The scripts below are used to build, test, and lint within the project.
|
||||
|
||||
- **Python**: [Dev Setup](./python/DEV_SETUP.md) · [Coding Standard](./python/CODING_STANDARD.md) · [README](./python/README.md)
|
||||
- From the `./python` directory:
|
||||
- Build: `uv run poe build`
|
||||
- Unit tests: `uv run poe test -A -m "not integration"`
|
||||
- Integration tests: `uv run poe test -A -m integration` (requires API keys/endpoints)
|
||||
- Format + lint: `uv run poe syntax`
|
||||
- All checks: `uv run poe check`
|
||||
- **.NET**: [README](./dotnet/README.md) · [Agent Instructions](./dotnet/AGENTS.md)
|
||||
- From the `./dotnet` directory:
|
||||
- Build: `dotnet build`
|
||||
- Unit tests: `dotnet test --filter-query "/*UnitTests*/*/*/*"`
|
||||
- Integration tests: `dotnet test --filter-query "/*IntegrationTests*/*/*/*"` (requires API keys/endpoints)
|
||||
- Linting (auto-fix): `dotnet format`
|
||||
- Python: see [python/DEV_SETUP.md](./python/DEV_SETUP.md).
|
||||
- .NET:
|
||||
- Build: `dotnet build`
|
||||
- Test: `dotnet test`
|
||||
- Linting (auto-fix): `dotnet format`
|
||||
|
||||
### PR - CI Process
|
||||
|
||||
|
||||
@@ -2,16 +2,12 @@
|
||||
|
||||
# Welcome to Microsoft Agent Framework!
|
||||
|
||||
[](https://discord.gg/b5zjErwbQM)
|
||||
[](https://discord.gg/b5zjErwbQM)
|
||||
[](https://learn.microsoft.com/en-us/agent-framework/)
|
||||
[](https://pypi.org/project/agent-framework/)
|
||||
[](https://www.nuget.org/profiles/MicrosoftAgentFramework/)
|
||||
[](https://github.com/microsoft/agent-framework)
|
||||
|
||||
|
||||
Microsoft Agent Framework (MAF) is an open, multi-language framework for building **production-grade AI agents and multi-agent workflows** in **.NET and Python**.
|
||||
|
||||
Microsoft Agent Framework is built for teams taking agents from prototype to production. It provides a consistent foundation for building, orchestrating, and operating agent systems across Python and .NET, while keeping architecture choices open as requirements evolve, and supports a broad ecosystem including Microsoft Foundry, Azure OpenAI, OpenAI, and the GitHub Copilot SDK, with samples and hosting patterns for both local development and cloud deployment.
|
||||
Welcome to Microsoft's comprehensive multi-language framework for building, orchestrating, and deploying AI agents with support for both .NET and Python implementations. This framework provides everything from simple chat agents to complex multi-agent workflows with graph-based orchestration.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.youtube.com/watch?v=AAgdMhftj8w" title="Watch the full Agent Framework introduction (30 min)">
|
||||
@@ -25,58 +21,14 @@ Microsoft Agent Framework is built for teams taking agents from prototype to pro
|
||||
</a>
|
||||
</p>
|
||||
|
||||
## Is this the right framework for you?
|
||||
## 📋 Getting Started
|
||||
|
||||
MAF is a strong fit if you:
|
||||
- are building agents and workflows you expect to run in production,
|
||||
- need orchestration beyond a single prompt or stateless chat loop,
|
||||
- want graph-based patterns such as sequential, concurrent, handoff, and group collaboration,
|
||||
- care about durability, restartability, observability, governance, or human-in-the-loop control,
|
||||
- need provider flexibility so your architecture can evolve without major rewrites.
|
||||
### 📦 Installation
|
||||
|
||||
## Key Features
|
||||
Explore new MAF capabilities and real implementation patterns on the [official blog](https://devblogs.microsoft.com/agent-framework/).
|
||||
|
||||
- **Python and C#/.NET Support**: Full framework support for both Python and C#/.NET implementations with consistent APIs
|
||||
- [Python packages](./python/packages/) | [.NET source](./dotnet/src/)
|
||||
- **Multiple Agent Provider Support**: Support for various LLM providers with more being added continuously
|
||||
- [Python examples](./python/samples/02-agents/providers/) | [.NET examples](./dotnet/samples/02-agents/AgentProviders/)
|
||||
- **Middleware**: Flexible middleware system for request/response processing, exception handling, and custom pipelines
|
||||
- [Python middleware](./python/samples/02-agents/middleware/) | [.NET middleware](./dotnet/samples/02-agents/Agents/Agent_Step11_Middleware/)
|
||||
- **Orchestration Patterns & Workflows**: Build multi-agent systems with graph-based workflows supporting sequential, concurrent, handoff, and group collaboration patterns; includes checkpointing, streaming, human-in-the-loop, and time-travel
|
||||
- [Python workflows](./python/samples/03-workflows/) | [.NET workflows](./dotnet/samples/03-workflows/)
|
||||
- **Foundry Hosted Agents (new)**: Deploy and host your agents to Foundry-hosted infrastructure with just 2 additional lines of code
|
||||
- [Python samples](./python/samples/04-hosting/foundry-hosted-agents/) | [.NET samples](./dotnet/samples/04-hosting/FoundryHostedAgents/)
|
||||
- **Observability**: Built-in OpenTelemetry integration for distributed tracing, monitoring, and debugging
|
||||
- [Python observability](./python/samples/02-agents/observability/) | [.NET telemetry](./dotnet/samples/02-agents/AgentOpenTelemetry/)
|
||||
- **Declarative Agents**: Define agents using YAML for faster setup and versioning
|
||||
- [Declarative agent samples](./declarative-agents/)
|
||||
- **Agent Skills**: Build domain-specific knowledge bases from multiple sources—files, inline code, class libraries—for agents to discover and use
|
||||
- [Skills design](./docs/decisions/0021-agent-skills-design.md)
|
||||
- **AF Labs**: Experimental packages for cutting-edge features including benchmarking, reinforcement learning, and research initiatives
|
||||
- [Labs directory](./python/packages/lab/)
|
||||
- **DevUI**: Interactive developer UI for agent development, testing, and debugging workflows
|
||||
- [See the DevUI in action](https://www.youtube.com/watch?v=mOAaGY4WPvc)
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Getting Started](#getting-started)
|
||||
- [Installation](#installation)
|
||||
- [Learning Resources](#learning-resources)
|
||||
- [Quickstart](#quickstart)
|
||||
- [Basic Agent - Python](#basic-agent---python)
|
||||
- [Basic Agent - .NET](#basic-agent---net)
|
||||
- [More Examples & Samples](#more-examples--samples)
|
||||
- [Community & Feedback](#community--feedback)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Contributor Resources](#contributor-resources)
|
||||
|
||||
## Getting Started
|
||||
### Installation
|
||||
Python
|
||||
|
||||
```bash
|
||||
pip install agent-framework
|
||||
pip install agent-framework --pre
|
||||
# This will install all sub-packages, see `python/packages` for individual packages.
|
||||
# It may take a minute on first install on Windows.
|
||||
```
|
||||
@@ -85,13 +37,9 @@ pip install agent-framework
|
||||
|
||||
```bash
|
||||
dotnet add package Microsoft.Agents.AI
|
||||
# For Foundry integration (used in the .NET quickstart below):
|
||||
dotnet add package Microsoft.Agents.AI.Foundry
|
||||
dotnet add package Azure.AI.Projects
|
||||
dotnet add package Azure.Identity
|
||||
```
|
||||
|
||||
### Learning Resources
|
||||
### 📚 Documentation
|
||||
|
||||
- **[Overview](https://learn.microsoft.com/agent-framework/overview/agent-framework-overview)** - High level overview of the framework
|
||||
- **[Quick Start](https://learn.microsoft.com/agent-framework/tutorials/quick-start)** - Get started with a simple agent
|
||||
@@ -100,34 +48,69 @@ dotnet add package Azure.Identity
|
||||
- **[Migration from Semantic Kernel](https://learn.microsoft.com/en-us/agent-framework/migration-guide/from-semantic-kernel)** - Guide to migrate from Semantic Kernel
|
||||
- **[Migration from AutoGen](https://learn.microsoft.com/en-us/agent-framework/migration-guide/from-autogen)** - Guide to migrate from AutoGen
|
||||
|
||||
### Quickstart
|
||||
Still have questions? Join our [weekly office hours](./COMMUNITY.md#public-community-office-hours) or ask questions in our [Discord channel](https://discord.gg/b5zjErwbQM) to get help from the team and other users.
|
||||
|
||||
#### Basic Agent - Python
|
||||
### ✨ **Highlights**
|
||||
|
||||
- **Graph-based Workflows**: Connect agents and deterministic functions using data flows with streaming, checkpointing, human-in-the-loop, and time-travel capabilities
|
||||
- [Python workflows](./python/samples/03-workflows/) | [.NET workflows](./dotnet/samples/03-workflows/)
|
||||
- **AF Labs**: Experimental packages for cutting-edge features including benchmarking, reinforcement learning, and research initiatives
|
||||
- [Labs directory](./python/packages/lab/)
|
||||
- **DevUI**: Interactive developer UI for agent development, testing, and debugging workflows
|
||||
- [DevUI package](./python/packages/devui/)
|
||||
|
||||
<p align="center">
|
||||
<a href="https://www.youtube.com/watch?v=mOAaGY4WPvc">
|
||||
<img src="https://img.youtube.com/vi/mOAaGY4WPvc/hqdefault.jpg" alt="See the DevUI in action" width="480">
|
||||
</a>
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://www.youtube.com/watch?v=mOAaGY4WPvc">
|
||||
See the DevUI in action (1 min)
|
||||
</a>
|
||||
</p>
|
||||
|
||||
- **Python and C#/.NET Support**: Full framework support for both Python and C#/.NET implementations with consistent APIs
|
||||
- [Python packages](./python/packages/) | [.NET source](./dotnet/src/)
|
||||
- **Observability**: Built-in OpenTelemetry integration for distributed tracing, monitoring, and debugging
|
||||
- [Python observability](./python/samples/02-agents/observability/) | [.NET telemetry](./dotnet/samples/02-agents/AgentOpenTelemetry/)
|
||||
- **Multiple Agent Provider Support**: Support for various LLM providers with more being added continuously
|
||||
- [Python examples](./python/samples/02-agents/providers/) | [.NET examples](./dotnet/samples/02-agents/AgentProviders/)
|
||||
- **Middleware**: Flexible middleware system for request/response processing, exception handling, and custom pipelines
|
||||
- [Python middleware](./python/samples/02-agents/middleware/) | [.NET middleware](./dotnet/samples/02-agents/Agents/Agent_Step11_Middleware/)
|
||||
|
||||
### 💬 **We want your feedback!**
|
||||
|
||||
- For bugs, please file a [GitHub issue](https://github.com/microsoft/agent-framework/issues).
|
||||
|
||||
## Quickstart
|
||||
|
||||
### Basic Agent - Python
|
||||
|
||||
Create a simple Azure Responses Agent that writes a haiku about the Microsoft Agent Framework
|
||||
|
||||
```python
|
||||
# pip install agent-framework
|
||||
# pip install agent-framework --pre
|
||||
# Use `az login` to authenticate with Azure CLI
|
||||
import os
|
||||
import asyncio
|
||||
from agent_framework import Agent
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from agent_framework.azure import AzureOpenAIResponsesClient
|
||||
from azure.identity import AzureCliCredential
|
||||
|
||||
|
||||
async def main():
|
||||
# Initialize a chat agent with Microsoft Foundry
|
||||
# Initialize a chat agent with Azure OpenAI Responses
|
||||
# the endpoint, deployment name, and api version can be set via environment variables
|
||||
# or they can be passed in directly to the FoundryChatClient constructor
|
||||
agent = Agent(
|
||||
client=FoundryChatClient(
|
||||
credential=AzureCliCredential(),
|
||||
# project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
# model=os.environ["FOUNDRY_MODEL_DEPLOYMENT_NAME"],
|
||||
),
|
||||
name="HaikuAgent",
|
||||
instructions="You are an upbeat assistant that writes beautifully.",
|
||||
# or they can be passed in directly to the AzureOpenAIResponsesClient constructor
|
||||
agent = AzureOpenAIResponsesClient(
|
||||
# endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
|
||||
# deployment_name=os.environ["AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME"],
|
||||
# api_version=os.environ["AZURE_OPENAI_API_VERSION"],
|
||||
# api_key=os.environ["AZURE_OPENAI_API_KEY"], # Optional if using AzureCliCredential
|
||||
credential=AzureCliCredential(), # Optional, if using api_key
|
||||
).as_agent(
|
||||
name="HaikuBot",
|
||||
instructions="You are an upbeat assistant that writes beautifully.",
|
||||
)
|
||||
|
||||
print(await agent.run("Write a haiku about Microsoft Agent Framework."))
|
||||
@@ -136,24 +119,43 @@ if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
#### Basic Agent - .NET
|
||||
Create a simple Agent, using Microsoft Foundry that writes a haiku about the Microsoft Agent Framework
|
||||
### Basic Agent - .NET
|
||||
|
||||
Create a simple Agent, using OpenAI Responses, that writes a haiku about the Microsoft Agent Framework
|
||||
|
||||
```c#
|
||||
// This sample shows how to create and run a basic agent with AIProjectClient.AsAIAgent(...).
|
||||
// dotnet add package Microsoft.Agents.AI.OpenAI --prerelease
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI;
|
||||
using OpenAI.Responses;
|
||||
|
||||
using Azure.AI.Projects;
|
||||
// Replace the <apikey> with your OpenAI API key.
|
||||
var agent = new OpenAIClient("<apikey>")
|
||||
.GetResponsesClient("gpt-4o-mini")
|
||||
.AsAIAgent(name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
|
||||
|
||||
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
|
||||
```
|
||||
|
||||
Create a simple Agent, using Azure OpenAI Responses with token based auth, that writes a haiku about the Microsoft Agent Framework
|
||||
|
||||
```c#
|
||||
// dotnet add package Microsoft.Agents.AI.OpenAI --prerelease
|
||||
// dotnet add package Azure.Identity
|
||||
// Use `az login` to authenticate with Azure CLI
|
||||
using System.ClientModel.Primitives;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI;
|
||||
using OpenAI.Responses;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
// Replace <resource> and gpt-4o-mini with your Azure OpenAI resource name and deployment name.
|
||||
var agent = new OpenAIClient(
|
||||
new BearerTokenPolicy(new AzureCliCredential(), "https://ai.azure.com/.default"),
|
||||
new OpenAIClientOptions() { Endpoint = new Uri("https://<resource>.openai.azure.com/openai/v1") })
|
||||
.GetResponsesClient("gpt-4o-mini")
|
||||
.AsAIAgent(name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
|
||||
|
||||
AIAgent agent =
|
||||
new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.AsAIAgent(model: deploymentName, instructions: "You are an upbeat assistant that writes beautifully.", name: "HaikuAgent");
|
||||
|
||||
// Once you have the agent, you can invoke it like any other AIAgent.
|
||||
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
|
||||
```
|
||||
|
||||
@@ -161,54 +163,23 @@ Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Fram
|
||||
|
||||
### Python
|
||||
|
||||
- [Getting Started](./python/samples/01-get-started): progressive tutorial from hello-world to workflows
|
||||
- [Getting Started with Agents](./python/samples/01-get-started): progressive tutorial from hello-world to hosting
|
||||
- [Agent Concepts](./python/samples/02-agents): deep-dive samples by topic (tools, middleware, providers, etc.)
|
||||
- [Workflows](./python/samples/03-workflows): workflow creation and integration with agents
|
||||
- [Hosting](./python/samples/04-hosting): A2A, self-hosted protocol helpers, and Foundry hosted agents. Durable Task and Azure Functions samples are in the [Durable Agent Framework extension](https://github.com/microsoft/agent-framework-durable-extension/tree/main/python/samples).
|
||||
- [End-to-End](./python/samples/05-end-to-end): full applications, evaluation, and demos
|
||||
- [Getting Started with Workflows](./python/samples/03-workflows): workflow creation and integration with agents
|
||||
|
||||
### .NET
|
||||
|
||||
- [Getting Started](./dotnet/samples/01-get-started): progressive tutorial from hello agent to workflows
|
||||
- [Agent Concepts](./dotnet/samples/02-agents/Agents): basic agent creation and tool usage
|
||||
- [Agent Providers](./dotnet/samples/02-agents/AgentProviders): samples showing different agent providers
|
||||
- [Workflows](./dotnet/samples/03-workflows): advanced multi-agent patterns and workflow orchestration
|
||||
- [Hosting](./dotnet/samples/04-hosting): A2A and Foundry hosted agents. Durable agent and workflow samples are in the [Durable Agent Framework extension](https://github.com/microsoft/agent-framework-durable-extension/tree/main/dotnet/samples).
|
||||
- [End-to-End](./dotnet/samples/05-end-to-end): full applications and demos
|
||||
|
||||
## Community & Feedback
|
||||
|
||||
- **Found a bug?** File a [GitHub issue](https://github.com/microsoft/agent-framework/issues) to help us improve.
|
||||
- **Enjoying MAF?** [](https://github.com/microsoft/agent-framework) to show your support and help others discover the project.
|
||||
- **Have questions?** Join our [Discord](https://discord.gg/b5zjErwbQM) or visit [weekly office hours](./COMMUNITY.md#public-community-office-hours).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Authentication
|
||||
|
||||
| Problem | Cause | Fix |
|
||||
|---------|-------|-----|
|
||||
| Authentication errors when using Azure credentials | Not signed in to Azure CLI | Run `az login` before starting your app |
|
||||
| API key errors | Wrong or missing API key | Verify the key and ensure it's for the correct resource/provider |
|
||||
|
||||
> **Tip:** `DefaultAzureCredential` is convenient for development but in production, consider using a specific credential (e.g., `ManagedIdentityCredential`) to avoid latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
|
||||
### Environment Variables
|
||||
For environment variable configuration specific to each sample, refer to the README in the sample directory ([Python samples](./python/samples/) | [.NET samples](./dotnet/samples/)).
|
||||
- [Getting Started with Agents](./dotnet/samples/02-agents/Agents): basic agent creation and tool usage
|
||||
- [Agent Provider Samples](./dotnet/samples/02-agents/AgentProviders): samples showing different agent providers
|
||||
- [Workflow Samples](./dotnet/samples/03-workflows): advanced multi-agent patterns and workflow orchestration
|
||||
|
||||
## Contributor Resources
|
||||
|
||||
- [Contributing Guide](./CONTRIBUTING.md)
|
||||
- [Code of Conduct](./CODE_OF_CONDUCT.md)
|
||||
- [Python Development Guide](./python/DEV_SETUP.md)
|
||||
- [Design Documents](./docs/design)
|
||||
- [Architectural Decision Records](./docs/decisions)
|
||||
|
||||
## Important Notes
|
||||
|
||||
> [!IMPORTANT]
|
||||
> If you use Microsoft Agent Framework to build applications that operate with any third-party servers, agents, code, or non-Azure Direct models (“Third-Party Systems”), you do so at your own risk. Third-Party Systems are Non-Microsoft Products under the Microsoft Product Terms and are governed by their own third-party license terms. You are responsible for any usage and associated costs.
|
||||
>
|
||||
>We recommend reviewing all data being shared with and received from Third-Party Systems and being cognizant of third-party practices for handling, sharing, retention and location of data. It is your responsibility to manage whether your data will flow outside of your organization’s Azure compliance and geographic boundaries and any related implications, and that appropriate permissions, boundaries and approvals are provisioned.
|
||||
>
|
||||
>You are responsible for carefully reviewing and testing applications you build using Microsoft Agent Framework in the context of your specific use cases, and making all appropriate decisions and customizations. This includes implementing your own responsible AI mitigations such as metaprompt, content filters, or other safety systems, and ensuring your applications meet appropriate quality, reliability, security, and trustworthiness standards. See also: [Transparency FAQ](./TRANSPARENCY_FAQ.md)
|
||||
If you use the Microsoft Agent Framework to build applications that operate with third-party servers or agents, you do so at your own risk. We recommend reviewing all data being shared with third-party servers or agents and being cognizant of third-party practices for retention and location of data. It is your responsibility to manage whether your data will flow outside of your organization's Azure compliance and geographic boundaries and any related implications.
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
# Support
|
||||
|
||||
## How to file issues and get help
|
||||
|
||||
This project uses GitHub Issues to track bugs and feature requests. Please search the existing
|
||||
issues before filing new issues to avoid duplicates. For new issues, file your bug or
|
||||
feature request as a new Issue.
|
||||
|
||||
For help and questions about using this project, please create a GitHub issue.
|
||||
|
||||
AI Support team will support Microsoft Agent Framework issues for customers under a **Unified support agreement when the issue arises from usage of Azure AI services** (Foundry Models, Foundry Agents etc.) in conjunction with the SDK. Conversely, if customer has any other / non unified support agreement and/or Agent Framework SDK is used in a way **not involving an Azure service**, it is treated as a purely open-source tool – Microsoft’s support organization will not handle it, and users should use GitHub or forums for assistance
|
||||
|
||||
For Copilot Studio SDK implementation issues, customers should use GitHub Issues for assistance, as outlined above. Conversely, for prerequisites managed within the Copilot Studio portal, customers can rely on the standard Microsoft Copilot Studio support channels.
|
||||
|
||||
## Microsoft Support Policy
|
||||
|
||||
Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
|
||||
# Support
|
||||
|
||||
## How to file issues and get help
|
||||
|
||||
This project uses GitHub Issues to track bugs and feature requests. Please search the existing
|
||||
issues before filing new issues to avoid duplicates. For new issues, file your bug or
|
||||
feature request as a new Issue.
|
||||
|
||||
For help and questions about using this project, please create a GitHub issue.
|
||||
|
||||
AI Support team will support Microsoft Agent Framework issues for customers under a **Unified support agreement when the issue arises from usage of Azure AI services** (Foundry Models, Foundry Agents etc.) in conjunction with the SDK. Conversely, if customer has any other / non unified support agreement and/or Agent Framework SDK is used in a way **not involving an Azure service**, it is treated as a purely open-source tool – Microsoft’s support organization will not handle it, and users should use GitHub or forums for assistance
|
||||
|
||||
For Copilot Studio SDK implementation issues, customers should use GitHub Issues for assistance, as outlined above. Conversely, for prerequisites managed within the Copilot Studio portal, customers can rely on the standard Microsoft Copilot Studio support channels.
|
||||
|
||||
## Microsoft Support Policy
|
||||
|
||||
Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
**What is Microsoft Agent Framework?**
|
||||
|
||||
Microsoft Agent Framework is a comprehensive multi-language (C#/.NET and Python) framework for building, orchestrating, and deploying AI agents and multi-agent workflows. The system takes user instructions and conversation inputs and produces intelligent responses through AI agents that can integrate with various LLM providers (OpenAI, Azure OpenAI, Microsoft Foundry). It provides both simple chat agents and complex multi-agent workflows with graph-based orchestration.
|
||||
Microsoft Agent Framework is a comprehensive multi-language (C#/.NET and Python) framework for building, orchestrating, and deploying AI agents and multi-agent workflows. The system takes user instructions and conversation inputs and produces intelligent responses through AI agents that can integrate with various LLM providers (OpenAI, Azure OpenAI, Azure AI Foundry). It provides both simple chat agents and complex multi-agent workflows with graph-based orchestration.
|
||||
|
||||
**What can Microsoft Agent Framework do?**
|
||||
|
||||
@@ -12,7 +12,7 @@ The framework offers:
|
||||
- **Multi-Agent Orchestration**: Group chat, sequential, concurrent, and handoff patterns
|
||||
- **Graph-based Workflows**: Connect agents and deterministic functions using data flows with streaming, checkpointing, time-travel, and Human-in-the-loop
|
||||
- **Extensibility Framework**: Extend with native functions, A2A, Model Context Protocol (MCP)
|
||||
- **LLM Integration**: Support for OpenAI, Azure OpenAI, Microsoft Foundry, and other providers
|
||||
- **LLM Integration**: Support for OpenAI, Azure OpenAI, Azure AI Foundry, and other providers
|
||||
- **Runtime Support**: Both in-process and distributed agent execution
|
||||
|
||||
**What is/are Microsoft Agent Framework's intended use(s)?**
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# Declarative Agents
|
||||
|
||||
This folder contains sample agent definitions that can be run using the declarative agent support, for python see the [declarative agent python sample folder](../python/samples/02-agents/declarative/).
|
||||
@@ -3,13 +3,13 @@ name: MicrosoftLearnAgent
|
||||
description: Microsoft Learn Agent
|
||||
instructions: You answer questions by searching the Microsoft Learn content only.
|
||||
model:
|
||||
id: =Env.FOUNDRY_MODEL
|
||||
id: =Env.AZURE_FOUNDRY_PROJECT_MODEL_ID
|
||||
options:
|
||||
temperature: 0.9
|
||||
topP: 0.95
|
||||
connection:
|
||||
kind: remote
|
||||
endpoint: =Env.FOUNDRY_PROJECT_ENDPOINT
|
||||
endpoint: =Env.AZURE_FOUNDRY_PROJECT_ENDPOINT
|
||||
tools:
|
||||
- kind: mcp
|
||||
name: microsoft_learn
|
||||
@@ -1,3 +0,0 @@
|
||||
# Declarative Agents
|
||||
|
||||
This folder contains sample agent definitions that can be run using the declarative agent support, for python see the [declarative agent python sample folder](../../python/samples/02-agents/declarative/).
|
||||
@@ -1,17 +0,0 @@
|
||||
# Declarative Workflows
|
||||
|
||||
A _Declarative Workflow_ is defined as a single YAML file and
|
||||
may be executed locally no different from any regular `Workflow` that is defined by code.
|
||||
|
||||
The difference is that the workflow definition is loaded from a YAML file instead of being defined in code:
|
||||
|
||||
```c#
|
||||
Workflow workflow = DeclarativeWorkflowBuilder.Build("Marketing.yaml", options);
|
||||
```
|
||||
|
||||
These example workflows may be executed by the workflow
|
||||
[Samples](../../dotnet/samples/03-workflows/Declarative)
|
||||
that are present in this repository.
|
||||
|
||||
> See the [README.md](../../dotnet/samples/03-workflows/Declarative/README.md)
|
||||
associated with the samples for configuration details.
|
||||
|
Before Width: | Height: | Size: 219 KiB |
|
Before Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 16 KiB |
|
Before Width: | Height: | Size: 19 KiB |
|
Before Width: | Height: | Size: 20 KiB |
@@ -1,55 +0,0 @@
|
||||
<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="256" height="256" transform="matrix(-1 0 0 1 256 0)" fill="white"/>
|
||||
<path d="M163.006 36.0262C155.129 31.4785 145.424 31.4759 137.533 36.0284L104.002 55.3877C107.838 53.2763 112.07 52.2274 116.296 52.2248C120.633 52.2231 124.976 53.3247 128.871 55.5411L174.091 81.6479C175.11 82.2362 175.738 83.3236 175.738 84.5004V151.263C175.738 152.716 177.313 153.621 178.568 152.895L196.967 142.28C204.845 137.732 209.704 129.326 209.704 120.221V77.7299C209.704 68.6342 204.855 60.227 196.967 55.6697L190.983 52.2141C190.65 52.0008 190.313 51.7921 189.969 51.5933L163.006 36.0262Z" fill="url(#paint0_linear_481_4810)"/>
|
||||
<path d="M163.006 36.0262C155.129 31.4785 145.424 31.4759 137.533 36.0284L104.002 55.3877C107.838 53.2763 112.07 52.2274 116.296 52.2248C120.633 52.2231 124.976 53.3247 128.871 55.5411L174.091 81.6479C175.11 82.2362 175.738 83.3236 175.738 84.5004V151.263C175.738 152.716 177.313 153.621 178.568 152.895L196.967 142.28C204.845 137.732 209.704 129.326 209.704 120.221V77.7299C209.704 68.6342 204.855 60.227 196.967 55.6697L190.983 52.2141C190.65 52.0008 190.313 51.7921 189.969 51.5933L163.006 36.0262Z" fill="url(#paint1_linear_481_4810)"/>
|
||||
<path d="M103.548 55.6397L103.557 55.6451L104.002 55.3877C103.851 55.471 103.698 55.5531 103.548 55.6397Z" fill="url(#paint2_linear_481_4810)"/>
|
||||
<path d="M103.548 55.6397L103.557 55.6451L104.002 55.3877C103.851 55.471 103.698 55.5531 103.548 55.6397Z" fill="url(#paint3_linear_481_4810)"/>
|
||||
<path d="M116.308 52.2209C111.903 52.2271 107.507 53.3498 103.561 55.6366C95.6702 60.1891 90.8239 68.6019 90.8231 77.6986L90.8223 167.846C90.8222 169.786 92.871 171.041 94.599 170.16L103.523 165.611C116.573 158.959 124.788 145.549 124.788 130.902V62.9894C124.778 58.6476 129.49 55.9209 133.25 58.0698L128.879 55.5453C124.976 53.3242 120.645 52.2192 116.308 52.2209Z" fill="url(#paint4_linear_481_4810)"/>
|
||||
<path d="M95.3068 221.682C103.184 226.23 112.889 226.232 120.78 221.68L154.311 202.32C150.475 204.432 146.243 205.481 142.018 205.483C137.68 205.485 133.337 204.383 129.442 202.167L84.2226 176.06C83.2035 175.472 82.5757 174.384 82.5757 173.208L82.5757 106.445C82.5757 104.992 81 104.087 79.7451 104.813L61.3465 115.428C53.4682 119.976 48.6091 128.382 48.6089 137.487L48.6089 179.978C48.6089 189.074 53.4585 197.481 61.3465 202.038L67.3303 205.494C67.6629 205.707 68.0002 205.916 68.3446 206.115L95.3068 221.682Z" fill="url(#paint5_linear_481_4810)"/>
|
||||
<path d="M95.3068 221.682C103.184 226.23 112.889 226.232 120.78 221.68L154.311 202.32C150.475 204.432 146.243 205.481 142.018 205.483C137.68 205.485 133.337 204.383 129.442 202.167L84.2226 176.06C83.2035 175.472 82.5757 174.384 82.5757 173.208L82.5757 106.445C82.5757 104.992 81 104.087 79.7451 104.813L61.3465 115.428C53.4682 119.976 48.6091 128.382 48.6089 137.487L48.6089 179.978C48.6089 189.074 53.4585 197.481 61.3465 202.038L67.3303 205.494C67.6629 205.707 68.0002 205.916 68.3446 206.115L95.3068 221.682Z" fill="url(#paint6_linear_481_4810)"/>
|
||||
<path d="M154.765 202.068L154.756 202.063L154.311 202.32C154.463 202.237 154.615 202.155 154.765 202.068Z" fill="url(#paint7_linear_481_4810)"/>
|
||||
<path d="M154.765 202.068L154.756 202.063L154.311 202.32C154.463 202.237 154.615 202.155 154.765 202.068Z" fill="url(#paint8_linear_481_4810)"/>
|
||||
<path d="M142.003 205.487C146.408 205.481 150.805 204.358 154.751 202.071C162.641 197.519 167.488 189.106 167.488 180.009L167.489 89.8618C167.489 87.9222 165.44 86.667 163.712 87.5479L154.788 92.0972C141.739 98.7494 133.523 112.159 133.523 126.806L133.523 194.719C133.533 199.06 128.821 201.787 125.061 199.638L129.432 202.163C133.336 204.384 137.666 205.489 142.003 205.487Z" fill="url(#paint9_linear_481_4810)"/>
|
||||
<defs>
|
||||
<linearGradient id="paint0_linear_481_4810" x1="207.413" y1="61.882" x2="148.999" y2="163.067" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#9189F7"/>
|
||||
<stop offset="1" stop-color="#4135E9"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint1_linear_481_4810" x1="188.235" y1="204.189" x2="264.21" y2="152.592" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#4F42FD"/>
|
||||
<stop offset="1" stop-color="#7274FF"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint2_linear_481_4810" x1="207.413" y1="61.882" x2="148.999" y2="163.067" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#9189F7"/>
|
||||
<stop offset="1" stop-color="#4135E9"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint3_linear_481_4810" x1="188.235" y1="204.189" x2="264.21" y2="152.592" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#4F42FD"/>
|
||||
<stop offset="1" stop-color="#7274FF"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint4_linear_481_4810" x1="93.1761" y1="128.826" x2="66.2399" y2="104.746" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.25" stop-color="#4F42FD"/>
|
||||
<stop offset="1" stop-color="#2C08AC"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint5_linear_481_4810" x1="50.9004" y1="195.826" x2="109.315" y2="94.6412" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#9189F7"/>
|
||||
<stop offset="1" stop-color="#4135E9"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint6_linear_481_4810" x1="70.0787" y1="53.5188" x2="-5.89657" y2="105.116" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#4F42FD"/>
|
||||
<stop offset="1" stop-color="#7274FF"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint7_linear_481_4810" x1="50.9004" y1="195.826" x2="109.315" y2="94.6412" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#9189F7"/>
|
||||
<stop offset="1" stop-color="#4135E9"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint8_linear_481_4810" x1="70.0787" y1="53.5188" x2="-5.89657" y2="105.116" gradientUnits="userSpaceOnUse">
|
||||
<stop stop-color="#4F42FD"/>
|
||||
<stop offset="1" stop-color="#7274FF"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="paint9_linear_481_4810" x1="165.135" y1="128.882" x2="192.072" y2="152.962" gradientUnits="userSpaceOnUse">
|
||||
<stop offset="0.25" stop-color="#4F42FD"/>
|
||||
<stop offset="1" stop-color="#2C08AC"/>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 5.8 KiB |
@@ -1,5 +0,0 @@
|
||||
<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="256" height="256" transform="matrix(-1 0 0 1 256 0)" fill="white"/>
|
||||
<path d="M167.489 89.8618C167.489 87.9224 165.441 86.667 163.713 87.5473L154.788 92.0971C141.739 98.7493 133.523 112.159 133.523 126.806V194.718C133.533 199.053 128.837 201.777 125.08 199.647L84.2227 176.06C83.2036 175.472 82.5762 174.384 82.5762 173.207V106.445C82.5759 104.992 80.9999 104.087 79.7451 104.813L61.3467 115.428C53.4685 119.976 48.6096 128.382 48.6094 137.487V179.978C48.6094 189.074 53.4588 197.481 61.3467 202.039L67.3301 205.494C67.6627 205.707 68.0004 205.916 68.3447 206.115L95.3066 221.682C103.184 226.23 112.889 226.232 120.779 221.679L154.312 202.32C154.271 202.342 154.23 202.362 154.189 202.384C154.283 202.333 154.375 202.281 154.468 202.229L154.312 202.32C154.463 202.237 154.615 202.154 154.765 202.068L154.762 202.065C162.646 197.511 167.487 189.102 167.488 180.009L167.489 89.8618Z" fill="black"/>
|
||||
<path d="M163.007 36.0259C155.13 31.4781 145.424 31.4763 137.533 36.0288L104.002 55.3882C104.029 55.3732 104.057 55.359 104.084 55.3442C104.02 55.3794 103.956 55.4159 103.893 55.4516L104.002 55.3882C103.851 55.4714 103.699 55.5536 103.549 55.6401L103.552 55.6411C95.6664 60.1948 90.8241 68.6053 90.8232 77.6987L90.8223 167.846C90.8223 169.786 92.8707 171.04 94.5986 170.16L103.523 165.611C116.573 158.959 124.788 145.549 124.788 130.902V62.9897C124.778 58.6479 129.49 55.9211 133.25 58.0698L174.091 81.6479C175.11 82.2363 175.737 83.3238 175.737 84.5005V151.263C175.738 152.716 177.314 153.621 178.568 152.895L196.967 142.28C204.845 137.732 209.704 129.326 209.704 120.221V77.73C209.704 68.6343 204.855 60.2267 196.967 55.6694L190.982 52.2143C190.65 52.0011 190.313 51.792 189.969 51.5932L163.007 36.0259Z" fill="black"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.8 KiB |
@@ -1,5 +0,0 @@
|
||||
<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="256" height="256" transform="matrix(-1 0 0 1 256 0)" fill="black"/>
|
||||
<path d="M167.489 89.8618C167.489 87.9224 165.441 86.667 163.713 87.5473L154.788 92.0971C141.739 98.7493 133.523 112.159 133.523 126.806V194.718C133.533 199.053 128.837 201.777 125.08 199.647L84.2227 176.06C83.2036 175.472 82.5762 174.384 82.5762 173.207V106.445C82.5759 104.992 80.9999 104.087 79.7451 104.813L61.3467 115.428C53.4685 119.976 48.6096 128.382 48.6094 137.487V179.978C48.6094 189.074 53.4588 197.481 61.3467 202.039L67.3301 205.494C67.6627 205.707 68.0004 205.916 68.3447 206.115L95.3066 221.682C103.184 226.23 112.889 226.232 120.779 221.679L154.312 202.32C154.271 202.342 154.23 202.362 154.189 202.384C154.283 202.333 154.375 202.281 154.468 202.229L154.312 202.32C154.463 202.237 154.615 202.154 154.765 202.068L154.762 202.065C162.646 197.511 167.487 189.102 167.488 180.009L167.489 89.8618Z" fill="white"/>
|
||||
<path d="M163.007 36.0259C155.13 31.4781 145.424 31.4763 137.533 36.0288L104.002 55.3882C104.029 55.3732 104.057 55.359 104.084 55.3442C104.02 55.3794 103.956 55.4159 103.893 55.4516L104.002 55.3882C103.851 55.4714 103.699 55.5536 103.549 55.6401L103.552 55.6411C95.6664 60.1948 90.8241 68.6053 90.8232 77.6987L90.8223 167.846C90.8223 169.786 92.8707 171.04 94.5986 170.16L103.523 165.611C116.573 158.959 124.788 145.549 124.788 130.902V62.9897C124.778 58.6479 129.49 55.9211 133.25 58.0698L174.091 81.6479C175.11 82.2363 175.737 83.3238 175.737 84.5005V151.263C175.738 152.716 177.314 153.621 178.568 152.895L196.967 142.28C204.845 137.732 209.704 129.326 209.704 120.221V77.73C209.704 68.6343 204.855 60.2267 196.967 55.6694L190.982 52.2143C190.65 52.0011 190.313 51.792 189.969 51.5932L163.007 36.0259Z" fill="white"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.8 KiB |
@@ -1,4 +0,0 @@
|
||||
<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="256" height="256" transform="matrix(-1 0 0 1 256 0)" fill="white"/>
|
||||
<path d="M48.6094 179.978V137.487C48.6096 128.382 53.4685 119.976 61.3467 115.428L79.7451 104.813C80.9999 104.087 82.5759 104.992 82.5762 106.445V173.207C82.5762 174.384 83.2036 175.472 84.2227 176.06L125.08 199.647C128.837 201.777 133.533 199.053 133.523 194.718V126.806C133.523 112.388 141.484 99.1684 154.18 92.4135L154.788 92.0971L163.713 87.5473C165.441 86.667 167.489 87.9224 167.489 89.8618L167.488 180.009L167.474 180.86C167.181 189.624 162.399 197.653 154.762 202.065L154.765 202.068C154.615 202.154 154.463 202.237 154.312 202.32L154.468 202.229C154.375 202.281 154.283 202.333 154.189 202.384C154.23 202.362 154.271 202.342 154.312 202.32L120.779 221.679L120.034 222.092C112.534 226.093 103.539 226.092 96.0508 222.095L95.3066 221.682L68.3447 206.115C68.0004 205.916 67.6627 205.707 67.3301 205.494L61.3467 202.039C53.7053 197.624 48.9149 189.595 48.623 180.829L48.6094 179.978ZM175.737 84.5005C175.737 83.3974 175.186 82.3728 174.277 81.7641L174.091 81.6479L133.25 58.0698C129.49 55.9211 124.778 58.6479 124.788 62.9897V130.902L124.782 131.587C124.53 145.966 116.369 159.063 103.523 165.611L94.5986 170.16L94.4355 170.236C92.799 170.938 90.9459 169.803 90.8281 168.026L90.8223 167.846L90.8232 77.6987C90.8241 68.6053 95.6664 60.1948 103.552 55.6411L103.549 55.6401C103.699 55.5536 103.851 55.4714 104.002 55.3882L103.893 55.4516C103.956 55.4159 104.02 55.3794 104.084 55.3442C104.057 55.359 104.029 55.3732 104.002 55.3882L137.533 36.0288C145.424 31.4763 155.13 31.4781 163.007 36.0259L189.969 51.5932C190.313 51.792 190.65 52.0011 190.982 52.2143L196.967 55.6694C204.855 60.2267 209.704 68.6343 209.704 77.73V120.221L209.689 121.073C209.397 129.848 204.599 137.874 196.967 142.28L178.568 152.895C177.314 153.621 175.738 152.716 175.737 151.263V84.5005ZM176.814 149.866L176.819 149.864L176.832 149.856C176.826 149.859 176.82 149.862 176.814 149.866ZM137.023 194.71L137.019 195.038C136.802 201.878 129.341 206.087 123.354 202.692L123.33 202.678L82.4727 179.091C80.3698 177.877 79.0762 175.634 79.0762 173.207V109.239L63.0957 118.458C56.5124 122.259 52.3745 129.183 52.1221 136.752L52.1094 137.487V179.978C52.1094 187.823 56.2909 195.075 63.0957 199.007H63.0967L69.0801 202.462L69.1504 202.503L69.2188 202.547C69.5212 202.741 69.8111 202.92 70.0947 203.083L97.0566 218.651L97.6982 219.007C104.372 222.569 112.434 222.453 119.029 218.648L152.514 199.316L152.512 199.312C152.581 199.274 152.65 199.236 152.752 199.178L152.753 199.181C152.775 199.169 152.796 199.158 152.815 199.147L153.011 199.035C159.81 195.107 163.987 187.854 163.988 180.009V91.3344L156.378 95.2153C144.501 101.27 137.023 113.475 137.023 126.806V194.71ZM94.3223 166.372L101.934 162.493C113.811 156.438 121.288 144.233 121.288 130.902V62.9897C121.278 55.9521 128.901 51.5532 134.986 55.0307L135 55.0385L175.841 78.6167L176.036 78.7339C178.023 79.9714 179.237 82.15 179.237 84.5005V148.468L195.217 139.249C202.013 135.326 206.204 128.075 206.204 120.221V77.73C206.204 69.8848 202.022 62.6318 195.217 58.6997L189.232 55.2456L189.162 55.2046L189.094 55.1606C188.788 54.9649 188.5 54.787 188.219 54.6245L161.257 39.0571C154.463 35.135 146.091 35.1311 139.282 39.0591L139.283 39.06L105.765 58.4106L105.767 58.4135C105.713 58.443 105.723 58.4383 105.602 58.5063L105.6 58.5034C105.535 58.5391 105.481 58.5691 105.433 58.5962L105.302 58.6723C98.5016 62.5995 94.324 69.8532 94.3232 77.6987L94.3223 166.372Z" fill="black"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.5 KiB |
@@ -1,4 +0,0 @@
|
||||
<svg width="256" height="256" viewBox="0 0 256 256" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="256" height="256" transform="matrix(-1 0 0 1 256 0)" fill="black"/>
|
||||
<path d="M48.6094 179.978V137.487C48.6096 128.382 53.4685 119.976 61.3467 115.428L79.7451 104.813C80.9999 104.087 82.5759 104.992 82.5762 106.445V173.207C82.5762 174.384 83.2036 175.472 84.2227 176.06L125.08 199.647C128.837 201.777 133.533 199.053 133.523 194.718V126.806C133.523 112.388 141.484 99.1684 154.18 92.4135L154.788 92.0971L163.713 87.5473C165.441 86.667 167.489 87.9224 167.489 89.8618L167.488 180.009L167.474 180.86C167.181 189.624 162.399 197.653 154.762 202.065L154.765 202.068C154.615 202.154 154.463 202.237 154.312 202.32L154.468 202.229C154.375 202.281 154.283 202.333 154.189 202.384C154.23 202.362 154.271 202.342 154.312 202.32L120.779 221.679L120.034 222.092C112.534 226.093 103.539 226.092 96.0508 222.095L95.3066 221.682L68.3447 206.115C68.0004 205.916 67.6627 205.707 67.3301 205.494L61.3467 202.039C53.7053 197.624 48.9149 189.595 48.623 180.829L48.6094 179.978ZM175.737 84.5005C175.737 83.3974 175.186 82.3728 174.277 81.7641L174.091 81.6479L133.25 58.0698C129.49 55.9211 124.778 58.6479 124.788 62.9897V130.902L124.782 131.587C124.53 145.966 116.369 159.063 103.523 165.611L94.5986 170.16L94.4355 170.236C92.799 170.938 90.9459 169.803 90.8281 168.026L90.8223 167.846L90.8232 77.6987C90.8241 68.6053 95.6664 60.1948 103.552 55.6411L103.549 55.6401C103.699 55.5536 103.851 55.4714 104.002 55.3882L103.893 55.4516C103.956 55.4159 104.02 55.3794 104.084 55.3442C104.057 55.359 104.029 55.3732 104.002 55.3882L137.533 36.0288C145.424 31.4763 155.13 31.4781 163.007 36.0259L189.969 51.5932C190.313 51.792 190.65 52.0011 190.982 52.2143L196.967 55.6694C204.855 60.2267 209.704 68.6343 209.704 77.73V120.221L209.689 121.073C209.397 129.848 204.599 137.874 196.967 142.28L178.568 152.895C177.314 153.621 175.738 152.716 175.737 151.263V84.5005ZM176.814 149.866L176.819 149.864L176.832 149.856C176.826 149.859 176.82 149.862 176.814 149.866ZM137.023 194.71L137.019 195.038C136.802 201.878 129.341 206.087 123.354 202.692L123.33 202.678L82.4727 179.091C80.3698 177.877 79.0762 175.634 79.0762 173.207V109.239L63.0957 118.458C56.5124 122.259 52.3745 129.183 52.1221 136.752L52.1094 137.487V179.978C52.1094 187.823 56.2909 195.075 63.0957 199.007H63.0967L69.0801 202.462L69.1504 202.503L69.2188 202.547C69.5212 202.741 69.8111 202.92 70.0947 203.083L97.0566 218.651L97.6982 219.007C104.372 222.569 112.434 222.453 119.029 218.648L152.514 199.316L152.512 199.312C152.581 199.274 152.65 199.236 152.752 199.178L152.753 199.181C152.775 199.169 152.796 199.158 152.815 199.147L153.011 199.035C159.81 195.107 163.987 187.854 163.988 180.009V91.3344L156.378 95.2153C144.501 101.27 137.023 113.475 137.023 126.806V194.71ZM94.3223 166.372L101.934 162.493C113.811 156.438 121.288 144.233 121.288 130.902V62.9897C121.278 55.9521 128.901 51.5532 134.986 55.0307L135 55.0385L175.841 78.6167L176.036 78.7339C178.023 79.9714 179.237 82.15 179.237 84.5005V148.468L195.217 139.249C202.013 135.326 206.204 128.075 206.204 120.221V77.73C206.204 69.8848 202.022 62.6318 195.217 58.6997L189.232 55.2456L189.162 55.2046L189.094 55.1606C188.788 54.9649 188.5 54.787 188.219 54.6245L161.257 39.0571C154.463 35.135 146.091 35.1311 139.282 39.0591L139.283 39.06L105.765 58.4106L105.767 58.4135C105.713 58.443 105.723 58.4383 105.602 58.5063L105.6 58.5034C105.535 58.5391 105.481 58.5691 105.433 58.5962L105.302 58.6723C98.5016 62.5995 94.324 69.8532 94.3232 77.6987L94.3223 166.372Z" fill="white"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.5 KiB |
|
Before Width: | Height: | Size: 1.5 MiB After Width: | Height: | Size: 136 KiB |
@@ -2,7 +2,7 @@
|
||||
# These are optional elements. Feel free to remove any of them.
|
||||
status: accepted
|
||||
contact: westey-m
|
||||
date: 2025-07-10
|
||||
date: 2025-07-10 {YYYY-MM-DD when the decision was last updated}
|
||||
deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub
|
||||
consulted:
|
||||
informed:
|
||||
@@ -139,7 +139,7 @@ Therefore something like `AgentResponse.Text` which also aggregates all `TextCon
|
||||
|
||||
#### Option 1.2 Presence of Secondary Content is determined by a runtime parameter
|
||||
|
||||
We can allow callers to choose whether to include secondary content in the list of response messages.
|
||||
We can allow callers to choose whether to include secondary content in the list of reponse messages.
|
||||
Open Question: Do we allow secondary content to use `TextContent` types?
|
||||
|
||||
```csharp
|
||||
|
||||
@@ -113,7 +113,7 @@ Implement a hybrid strategy where common tools use generic `AITool`-derived abst
|
||||
|
||||
### AI Agent Tool Types Availability
|
||||
|
||||
Tool Type | Microsoft Foundry Agent Service | OpenAI Assistant API | OpenAI ChatCompletion API | OpenAI Responses API | Amazon Bedrock Agents | Google | Anthropic | Description
|
||||
Tool Type | Azure AI Foundry Agent Service | OpenAI Assistant API | OpenAI ChatCompletion API | OpenAI Responses API | Amazon Bedrock Agents | Google | Anthropic | Description
|
||||
-- | -- | -- | -- | -- | -- | -- | -- | --
|
||||
Function Calling | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | Enables custom, stateless functions to define specific agent behaviors.
|
||||
Code Interpreter | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | Allows agents to execute code for tasks like data analysis or problem-solving.
|
||||
@@ -132,7 +132,7 @@ Image Generation | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | Generates or edits
|
||||
|
||||
#### Function Calling
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agent Service</summary>
|
||||
<summary>Azure AI Foundry Agent Service</summary>
|
||||
Source: <a href="https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/function-calling?pivots=rest">https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/function-calling?pivots=rest</a>
|
||||
|
||||
Message Request:
|
||||
@@ -401,7 +401,7 @@ Image Generation | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | Generates or edits
|
||||
|
||||
#### Code Interpreter
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agent Service</summary>
|
||||
<summary>Azure AI Foundry Agent Service</summary>
|
||||
<p>Source: <a href="https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/code-interpreter-samples?pivots=rest-api">https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/code-interpreter-samples?pivots=rest-api</a></p>
|
||||
|
||||
<p>.NET Support: ✅</p>
|
||||
@@ -709,7 +709,7 @@ Image Generation | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | Generates or edits
|
||||
|
||||
#### Search and Retrieval
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agent Service</summary>
|
||||
<summary>Azure AI Foundry Agent Service</summary>
|
||||
Source: <a href="https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/file-search-upload-files?pivots=rest">https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/file-search-upload-files?pivots=rest</a>
|
||||
|
||||
File Search Request:
|
||||
@@ -1083,7 +1083,7 @@ Image Generation | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | Generates or edits
|
||||
|
||||
#### Web Search
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agent Service</summary>
|
||||
<summary>Azure AI Foundry Agent Service</summary>
|
||||
Source: <a href="https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-code-samples?pivots=rest">https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-code-samples?pivots=rest</a>
|
||||
|
||||
Bing Search Message Request:
|
||||
@@ -1630,7 +1630,7 @@ Image Generation | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | Generates or edits
|
||||
|
||||
#### OpenAPI Spec Tool
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agent Service</summary>
|
||||
<summary>Azure AI Foundry Agent Service</summary>
|
||||
Source: <a href="https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/openapi-spec-samples?pivots=rest-api">https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/openapi-spec-samples?pivots=rest-api</a><br>
|
||||
Source: <a href="https://learn.microsoft.com/en-us/rest/api/aifoundry/aiagents/run-steps/get-run-step?view=rest-aifoundry-aiagents-v1&tabs=HTTP#runstepopenapitoolcall">https://learn.microsoft.com/en-us/rest/api/aifoundry/aiagents/run-steps/get-run-step?view=rest-aifoundry-aiagents-v1&tabs=HTTP#runstepopenapitoolcall</a>
|
||||
|
||||
@@ -1712,7 +1712,7 @@ Image Generation | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | Generates or edits
|
||||
|
||||
#### Stateful Functions
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agent Service</summary>
|
||||
<summary>Azure AI Foundry Agent Service</summary>
|
||||
Source: <a href="https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/azure-functions-samples?pivots=rest">https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/azure-functions-samples?pivots=rest</a>
|
||||
|
||||
Message Request:
|
||||
@@ -1832,7 +1832,7 @@ Image Generation | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | Generates or edits
|
||||
|
||||
#### Microsoft Fabric
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agent Service</summary>
|
||||
<summary>Azure AI Foundry Agent Service</summary>
|
||||
Source: <a href="https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/fabric?pivots=rest">https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/fabric?pivots=rest</a>
|
||||
|
||||
Message Request:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# These are optional elements. Feel free to remove any of them.
|
||||
status: accepted
|
||||
contact: westey-m
|
||||
date: 2025-09-12
|
||||
date: 2025-09-12 {YYYY-MM-DD when the decision was last updated}
|
||||
deciders: sergeymenshykh, markwallace-microsoft, rogerbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub, peterychang
|
||||
consulted:
|
||||
informed:
|
||||
@@ -25,7 +25,7 @@ See various features that would need to be supported via this type of mechanism,
|
||||
- Also see [the openai human-in-the-loop guide](https://openai.github.io/openai-agents-js/guides/human-in-the-loop/#approval-requests).
|
||||
- Also see [the openai MCP guide](https://openai.github.io/openai-agents-js/guides/mcp/#optional-approval-flow).
|
||||
- Also see [MCP Approval Requests from OpenAI](https://platform.openai.com/docs/guides/tools-remote-mcp#approvals).
|
||||
- Also see [Microsoft Foundry MCP Approvals](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/model-context-protocol-samples?pivots=rest#submit-your-approval).
|
||||
- Also see [Azure AI Foundry MCP Approvals](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/model-context-protocol-samples?pivots=rest#submit-your-approval).
|
||||
- Also see [MCP Elicitation requests](https://modelcontextprotocol.io/specification/draft/client/elicitation)
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
@@ -1125,7 +1125,7 @@ Naming (Python): N/A (Composable Components)
|
||||
Supports: N
|
||||
Observation: No explicit middleware/filters; modularity allows composable units but no dedicated interception hooks or callbacks for custom reading/modification mid-execution.
|
||||
|
||||
No specific code examples available for interception.
|
||||
For more details, see the official documentation: [Atomic Agents Docs](https://brainblend-ai.github.io/atomic-agents/). No specific code examples available for interception.
|
||||
|
||||
#### Smolagents (Hugging Face)
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ This section describes different options for various aspects required to add lon
|
||||
|
||||
### 1. Methods for Working with Long-Running Operations
|
||||
|
||||
Based on the analysis of existing APIs that support long-running operations (such as OpenAI Responses, Microsoft Foundry Agents, and A2A),
|
||||
Based on the analysis of existing APIs that support long-running operations (such as OpenAI Responses, Azure AI Foundry Agents, and A2A),
|
||||
the following operations are used for working with long-running operations:
|
||||
- Common operations:
|
||||
- **Start Long-Running Execution**: Initiates a long-running operation and returns its Id.
|
||||
@@ -757,7 +757,7 @@ Some of them natively support resuming streaming from a specific point in the st
|
||||
| API | Can Resume Streaming | Model |
|
||||
|-------------------------|--------------------------------------|------------------------------------------------------------------------------------------------------------|
|
||||
| OpenAI Responses | Yes | StreamingResponseUpdate.**SequenceNumber** + GetResponseStreamingAsync(responseId, **startingAfter**, ct) |
|
||||
| Microsoft Foundry Agents | Emulated<sup>2</sup> | RunStep.**Id** + custom pseudo code: client.Runs.GetRunStepsAsync(...).AllStepsAfter(**stepId**) |
|
||||
| Azure AI Foundry Agents | Emulated<sup>2</sup> | RunStep.**Id** + custom pseudo code: client.Runs.GetRunStepsAsync(...).AllStepsAfter(**stepId**) |
|
||||
| A2A | Implementation dependent<sup>1</sup> | |
|
||||
|
||||
<sup>1</sup> The [A2A specification](https://github.com/a2aproject/A2A/blob/main/docs/topics/streaming-and-async.md#1-streaming-with-server-sent-events-sse)
|
||||
@@ -765,7 +765,7 @@ allows an A2A agent implementation to decide how to handle streaming resumption:
|
||||
a task is still active (and the server hasn't sent a final: true event for that phase), the client can attempt to reconnect to the stream using the tasks/resubscribe RPC method.
|
||||
The server's behavior regarding missed events during the disconnection period (e.g., whether it backfills or only sends new updates) is implementation-dependent._
|
||||
|
||||
<sup>2</sup> The Microsoft Foundry Agents API has an API to start a streaming run but does not have an API to resume streaming from a specific point in the stream.
|
||||
<sup>2</sup> The Azure AI Foundry Agents API has an API to start a streaming run but does not have an API to resume streaming from a specific point in the stream.
|
||||
However, it has non-streaming APIs to access already started runs, which can be used to emulate streaming resumption by accessing a run and its steps and streaming all the steps after a specific step.
|
||||
|
||||
#### Required Changes
|
||||
@@ -828,7 +828,7 @@ Sequence of updates from OpenAI Responses API to answer the question "What time
|
||||
| resp_2 | 10 | resp.output_item.done | - | InProgress | |
|
||||
| resp_2 | 11 | resp.completed | Completed | Completed | |
|
||||
|
||||
Sequence of updates from Microsoft Foundry Agents API to answer the question "What time is it?" using a function call:
|
||||
Sequence of updates from Azure AI Foundry Agents API to answer the question "What time is it?" using a function call:
|
||||
| Id | SN | UpdateKind | Run.Status | Step.Status | Message.Status | ChatResponseUpdate.Status | Description |
|
||||
|--------|---------|-------------------|----------------|-------------|-----------------|---------------------------|---------------------------------------------------|
|
||||
| run_1 | - | RunCreated | Queued | - | - | Queued | |
|
||||
@@ -852,7 +852,7 @@ Sequence of updates from Microsoft Foundry Agents API to answer the question "Wh
|
||||
|
||||
To support long-running operations, the following values need to be returned by the GetResponseAsync and GetStreamingResponseAsync methods:
|
||||
- `ResponseId` - identifier of the long-running operation or an entity representing it, such as a task.
|
||||
- `ConversationId` - identifier of the conversation or thread the long-running operation is part of. Some APIs, like Microsoft Foundry Agents, use
|
||||
- `ConversationId` - identifier of the conversation or thread the long-running operation is part of. Some APIs, like Azure AI Foundry Agents, use
|
||||
this identifier together with the ResponseId to identify a run.
|
||||
- `SequenceNumber` - identifier of an update within a stream of updates. This is required to support streaming resumption by the GetStreamingResponseAsync method only.
|
||||
- `Status` - status of the long-running operation: whether it is queued, running, failed, cancelled, completed, etc.
|
||||
@@ -1089,7 +1089,7 @@ public class ChatOptions
|
||||
|
||||
##### 6.1.5 Continuation Token of a Custom Type
|
||||
|
||||
The option is similar to the "6.1.3 Continuation Token of System.ClientModel.ContinuationToken Type" option but suggests using a
|
||||
The option is similar the the "6.1.3 Continuation Token of System.ClientModel.ContinuationToken Type" option but suggests using a
|
||||
custom type for the continuation token instead of the `System.ClientModel.ContinuationToken` type.
|
||||
|
||||
**Pros**
|
||||
@@ -1203,7 +1203,7 @@ response = await agent.CancelRunAsync(response.ResponseId, new AgentCancelRunOpt
|
||||
In case an agent supports either or both cancellation and deletion of long-running operations, it will override the corresponding methods.
|
||||
Otherwise, it won't override them, and the base implementations will return null by default.
|
||||
|
||||
Some agents, for example Microsoft Foundry Agents, require the thread identifier to cancel a run. To accommodate this requirement, the `CancelRunAsync` method
|
||||
Some agents, for example Azure AI Foundry Agents, require the thread identifier to cancel a run. To accommodate this requirement, the `CancelRunAsync` method
|
||||
accepts an optional `AgentCancelRunOptions` parameter that allows callers to specify the thread associated with the run they want to cancel.
|
||||
|
||||
```csharp
|
||||
@@ -1574,7 +1574,7 @@ the thread is provided with background operations consistently for all runs.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Microsoft Foundry Agents</summary>
|
||||
<summary>Azure AI Foundry Agents</summary>
|
||||
|
||||
- Create a thread and run the agent against it and wait for it to complete using polling:
|
||||
```csharp
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
---
|
||||
status: accepted
|
||||
contact: rogerbarreto
|
||||
date: 2026-03-06
|
||||
deciders: rogerbarreto, alliscode
|
||||
consulted: ""
|
||||
informed: ""
|
||||
---
|
||||
|
||||
# Foundry agent surface stays centered on `ChatClientAgent`
|
||||
|
||||
## Context
|
||||
|
||||
The Microsoft Foundry integration exposes two distinct usage patterns:
|
||||
|
||||
1. Direct Responses usage, where callers provide model, instructions, and tools at runtime.
|
||||
2. Server-side versioned agents, where callers create and manage `AgentVersion` resources through `AIProjectClient.Agents`.
|
||||
|
||||
We briefly explored adding public wrapper types such as `FoundryAgent`, `FoundryVersionedAgent`, and `FoundryResponsesChatClient` to make those paths feel more specialized. That direction created extra public types, duplicated existing `ChatClientAgent` behavior, and pushed samples toward compatibility helpers instead of the native Azure SDK flow.
|
||||
|
||||
## Decision
|
||||
|
||||
Keep the public surface centered on `ChatClientAgent`.
|
||||
|
||||
- Direct Responses scenarios use `AIProjectClient.AsAIAgent(...)`.
|
||||
- Server-side versioned scenarios use native `AIProjectClient.Agents` APIs to create or retrieve agent resources, then wrap `AgentRecord` or `AgentVersion` with `AIProjectClient.AsAIAgent(...)`.
|
||||
- Compatibility helpers such as `AIProjectClient.CreateAIAgentAsync(...)` and `AIProjectClient.GetAIAgentAsync(...)` remain only as obsolete migration shims.
|
||||
- Public wrapper types `FoundryAgent`, `FoundryVersionedAgent`, `FoundryResponsesChatClient`, and `FoundryResponsesChatClientAgent` are not part of the chosen direction.
|
||||
|
||||
## Why
|
||||
|
||||
- `ChatClientAgent` is already the framework abstraction used everywhere else.
|
||||
- `AIProjectClient` is the native Azure SDK entry point for versioned agent lifecycle operations.
|
||||
- A single agent abstraction avoids parallel type hierarchies for the same backend.
|
||||
- Samples become clearer when they show either:
|
||||
- direct Responses construction via `AIProjectClient.AsAIAgent(...)`, or
|
||||
- native Foundry resource management via `AIProjectClient.Agents`.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Direct Responses path
|
||||
|
||||
Use the convenience overloads on `AIProjectClient`:
|
||||
|
||||
```csharp
|
||||
AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
|
||||
|
||||
ChatClientAgent agent = aiProjectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You are good at telling jokes.",
|
||||
name: "JokerAgent");
|
||||
```
|
||||
|
||||
Or use composed `ChatClientAgent`
|
||||
|
||||
```csharp
|
||||
ProjectResponsesClient projectResponsesClient = new(new Uri(endpoint), new DefaultAzureCredential(), new AgentReference($"model:{deploymentName}"));
|
||||
|
||||
ChatClientAgent agent = new(
|
||||
chatClient: projectResponsesClient.AsIChatClient(),
|
||||
instructions: "You are good at telling jokes.",
|
||||
name: "JokerAgent");
|
||||
```
|
||||
|
||||
This path is code-first and does not create a persistent server-side agent.
|
||||
|
||||
### Versioned agent path
|
||||
|
||||
Use the convenience overloads on `AIProjectClient`:
|
||||
|
||||
```csharp
|
||||
AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
|
||||
|
||||
AgentVersion version = await aiProjectClient.Agents.CreateAgentVersionAsync(
|
||||
"JokerAgent",
|
||||
new AgentVersionCreationOptions(
|
||||
new PromptAgentDefinition(deploymentName)
|
||||
{
|
||||
Instructions = "You are good at telling jokes."
|
||||
}));
|
||||
|
||||
ChatClientAgent agent = aiProjectClient.AsAIAgent(version);
|
||||
```
|
||||
|
||||
Or use composed `ChatClientAgent`
|
||||
|
||||
```csharp
|
||||
AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
|
||||
|
||||
AgentVersion version = await aiProjectClient.Agents.CreateAgentVersionAsync(
|
||||
"JokerAgent",
|
||||
new AgentVersionCreationOptions(
|
||||
new PromptAgentDefinition(deploymentName)
|
||||
{
|
||||
Instructions = "You are good at telling jokes."
|
||||
}));
|
||||
|
||||
ProjectResponsesClient projectResponsesClient = aiProjectClient
|
||||
.GetProjectOpenAIClient()
|
||||
.GetProjectResponsesClientForAgent(new AgentReference(version.Name, version.Version));
|
||||
|
||||
ChatClientAgent agent = new(
|
||||
chatClient: projectResponsesClient.AsIChatClient(),
|
||||
name: "JokerAgent");
|
||||
```
|
||||
|
||||
### Samples
|
||||
|
||||
- `FoundryAgents/` samples show the direct Responses path with `AIProjectClient.AsAIAgent(...)`.
|
||||
- `FoundryVersionedAgents/` samples should show native `AIProjectClient.Agents` create/get/delete flows plus `AsAIAgent(...)`.
|
||||
|
||||
### Compatibility APIs
|
||||
|
||||
Obsolete helper extensions remain only to ease migration of existing code. New samples and new guidance should not be written against them.
|
||||
|
||||
## Rejected direction
|
||||
|
||||
Do not introduce or preserve separate public wrapper types whose main purpose is to forward to `ChatClientAgent` while carrying Foundry-specific naming.
|
||||
|
||||
That approach:
|
||||
|
||||
- duplicates lifecycle concepts already present on `AIProjectClient`,
|
||||
- fragments the public API,
|
||||
- complicates samples and docs,
|
||||
- and makes migration harder by encouraging wrapper-specific affordances.
|
||||
@@ -0,0 +1,815 @@
|
||||
---
|
||||
status: accepted
|
||||
contact: bentho
|
||||
date: 2026-02-27
|
||||
deciders: bentho, markwallace-microsoft, westey-m
|
||||
consulted: Pratyush Mishra, Shivam Shrivastava, Manni Arora (Centrica eval scenario)
|
||||
informed: Agent Framework team, Foundry Evals team
|
||||
---
|
||||
|
||||
# Agent Evaluation Architecture with Azure AI Foundry Integration
|
||||
|
||||
## Context and Problem Statement
|
||||
|
||||
Azure AI Foundry provides a rich evaluation service for AI agents — built-in evaluators for agent behavior (task adherence, intent resolution), tool usage (tool call accuracy, tool selection), quality (coherence, fluency, relevance), and safety (violence, self-harm, prohibited actions). Results are viewable in the Foundry portal with dashboards and comparison views.
|
||||
|
||||
However, using Foundry Evals with an agent-framework agent today requires significant manual effort. Developers must:
|
||||
|
||||
1. Transform agent-framework's `Message`/`Content` types into the OpenAI-style agent message schema that Foundry evaluators expect
|
||||
2. Map tool definitions from agent-framework's `FunctionTool` format to evaluator-compatible schemas
|
||||
3. Manually wire up the correct Foundry data source type (`azure_ai_traces`, `jsonl`, `azure_ai_target_completions`, etc.) depending on their scenario
|
||||
4. Handle App Insights trace ID queries, response ID collection, and eval polling
|
||||
|
||||
Additionally, evaluation is a concern that extends beyond any single provider. Developers may want to use local evaluators (LLM-as-judge, regex, keyword matching), third-party evaluation libraries, or multiple providers in combination. The architecture must support this without creating a Foundry-specific lock-in at the API level.
|
||||
|
||||
### Functional Requirements for Agent Evaluation
|
||||
|
||||
- **Single agents and workflows.** Evaluate both individual agent responses and multi-agent workflow results, with per-agent breakdown to pinpoint underperformance.
|
||||
- **One-shot and multi-turn conversations.** Capture full conversation trajectories — including tool calls and results — not just final query/response pairs.
|
||||
- **Conversation factoring.** Support splitting conversations into query/response in multiple ways (last turn, full trajectory, per-turn) because different factorings measure different things.
|
||||
- **Multiple providers, mix and match.** Run Foundry LLM-as-judge evaluators alongside fast local checks and custom evaluators on the same data, without restructuring code.
|
||||
- **Third-party extensibility.** Any evaluation library can participate by implementing the `Evaluator` protocol (Python) or `IAgentEvaluator` interface (.NET). No predetermined list of supported libraries — the protocol is intentionally simple (`evaluate(items) → results`) so that wrappers for libraries like DeepEval, RAGAS, or Promptfoo are straightforward to write.
|
||||
- **Bring your own evaluator.** Creating a custom evaluator should be as simple as writing a function.
|
||||
- **Evaluate without re-running.** Evaluate existing responses from logs or previous runs without invoking the agent again.
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- **Zero-friction evaluation**: Developers should go from "I have an agent" to "I have eval results" with minimal code.
|
||||
- **Provider-agnostic API**: Core evaluation capabilities must not be tied to any specific provider. Provider configuration should be separate from the evaluation call.
|
||||
- **Lowest concept count**: Introduce the fewest possible new types, abstractions, and APIs for developers to learn.
|
||||
- **Leverage existing knowledge**: The framework already knows which agents exist, what tools they have, and what conversations occurred. Evals should use this automatically rather than requiring the developer to re-specify it.
|
||||
- **Foundry-native results**: When using Foundry, results should be viewable in the Foundry portal with dashboards and comparison views.
|
||||
- **Progressive disclosure**: Simple scenarios should be near-zero code. Advanced scenarios should build on the same primitives.
|
||||
- **Cross-language parity**: Design must be implementable in both Python and .NET.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Provider-specific functions** — Build Foundry-specific helper functions (`evaluate_agent()`, etc.) directly in the Azure package. All eval functions take Foundry connection parameters.
|
||||
2. **Evaluator protocol with shared orchestration** — Define a provider-agnostic `Evaluator` protocol in the base agent library (`agent_framework` in Python, `Microsoft.Agents.AI` in .NET). Orchestration functions live alongside it. Providers implement the protocol.
|
||||
3. **Full eval framework** — Build comprehensive eval infrastructure including custom evaluator definitions, scoring profiles, and reporting inside agent-framework.
|
||||
|
||||
## Decision Outcome
|
||||
|
||||
Proposed option: "Evaluator protocol with shared orchestration", because it delivers the low-friction developer experience, supports multiple providers without API changes, and keeps the concept count low.
|
||||
|
||||
### Usage Examples
|
||||
|
||||
#### Evaluate an agent
|
||||
|
||||
The agent is invoked once per query by default. For statistically meaningful evaluation, provide multiple diverse queries. For measuring **consistency** (does the same query produce reliable results?), use `num_repetitions` to run each query N times independently:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
evals = FoundryEvals(
|
||||
project_client=client,
|
||||
model_deployment="gpt-4o",
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=my_agent,
|
||||
queries=[
|
||||
"What's the weather in Seattle?",
|
||||
"Plan a weekend trip to Portland",
|
||||
"What restaurants are near Pike Place?",
|
||||
],
|
||||
evaluators=evals,
|
||||
)
|
||||
for r in results:
|
||||
r.assert_passed()
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var evals = new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] {
|
||||
"What's the weather in Seattle?",
|
||||
"Plan a weekend trip to Portland",
|
||||
"What restaurants are near Pike Place?",
|
||||
},
|
||||
evals);
|
||||
|
||||
results.AssertAllPassed();
|
||||
```
|
||||
|
||||
`evaluate_agent` returns one `EvalResults` per evaluator. Each result contains per-item scores with the evaluated response for auditing:
|
||||
|
||||
```
|
||||
# results[0] (FoundryEvals)
|
||||
EvalResults(status="completed", passed=3, failed=0, total=3)
|
||||
items[0]: EvalItemResult(
|
||||
query="What's the weather in Seattle?",
|
||||
response="It's currently 72°F and sunny in Seattle.",
|
||||
scores={"relevance": 5, "coherence": 5})
|
||||
items[1]: EvalItemResult(
|
||||
query="Plan a weekend trip to Portland",
|
||||
response="Here's a 2-day Portland itinerary...",
|
||||
scores={"relevance": 4, "coherence": 5})
|
||||
items[2]: EvalItemResult(
|
||||
query="What restaurants are near Pike Place?",
|
||||
response="Top restaurants near Pike Place Market: ...",
|
||||
scores={"relevance": 5, "coherence": 4})
|
||||
```
|
||||
|
||||
#### Measure consistency with repetitions
|
||||
|
||||
Run each query multiple times to detect non-deterministic behavior:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
results = await evaluate_agent(
|
||||
agent=my_agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=evals,
|
||||
num_repetitions=3, # each query runs 3 times independently
|
||||
)
|
||||
# results contain 3 items (1 query × 3 repetitions)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] { "What's the weather in Seattle?" },
|
||||
evals,
|
||||
numRepetitions: 3); // each query runs 3 times independently
|
||||
// results contain 3 items (1 query × 3 repetitions)
|
||||
```
|
||||
|
||||
#### Evaluate a response you already have
|
||||
|
||||
When you already have agent responses, pass them directly to skip re-running the agent. Each query is paired with its corresponding response:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
queries = ["What's the weather?", "What's the capital of France?"]
|
||||
responses = [await agent.run([Message("user", [q])]) for q in queries]
|
||||
|
||||
results = await evaluate_agent(
|
||||
responses=responses,
|
||||
evaluators=evals,
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var queries = new[] { "What's the weather?" };
|
||||
var responses = new List<AgentResponse>();
|
||||
foreach (var q in queries)
|
||||
responses.Add(await agent.RunAsync(new[] { new ChatMessage(ChatRole.User, q) }));
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
responses: responses,
|
||||
evals);
|
||||
```
|
||||
|
||||
Each `AgentResponse` already contains the conversation (query + response), so the evaluator extracts query/response from the conversation. When you pass `responses` without `queries`, the conversation is the source of truth.
|
||||
|
||||
#### Evaluate with conversation split strategies
|
||||
|
||||
By default, evaluators see only the last turn (final user message → final assistant response). For multi-turn conversations, you can control how the conversation is factored for evaluation:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["Plan a 3-day trip to Paris"],
|
||||
evaluators=evals,
|
||||
conversation_split=ConversationSplit.FULL, # evaluate entire trajectory
|
||||
)
|
||||
|
||||
# Or per-turn: each user→assistant exchange scored independently
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["Plan a 3-day trip to Paris"],
|
||||
evaluators=evals,
|
||||
conversation_split=ConversationSplit.PER_TURN,
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
// Full conversation as context
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] { "Plan a 3-day trip to Paris" },
|
||||
evals,
|
||||
splitter: ConversationSplitters.Full);
|
||||
|
||||
// Per-turn splitting
|
||||
var items = EvalItem.PerTurnItems(conversation); // one EvalItem per user turn
|
||||
var results = await evals.EvaluateAsync(items);
|
||||
```
|
||||
|
||||
With `PER_TURN`, a 3-turn conversation produces 3 scored items:
|
||||
|
||||
```
|
||||
EvalResults(status="completed", passed=3, failed=0, total=3)
|
||||
items[0]: query="Plan a 3-day trip to Paris" scores={"relevance": 5}
|
||||
items[1]: query="What about restaurants?" scores={"relevance": 4}
|
||||
items[2]: query="Make it budget-friendly" scores={"relevance": 5}
|
||||
```
|
||||
|
||||
#### Evaluate a multi-agent workflow
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
result = await workflow.run("Plan a trip to Paris")
|
||||
eval_results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
workflow_result=result,
|
||||
evaluators=evals,
|
||||
)
|
||||
|
||||
for r in eval_results:
|
||||
print(f" overall: {r.passed}/{r.total}")
|
||||
for name, sub in r.sub_results.items():
|
||||
print(f" {name}: {sub.passed}/{sub.total}")
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
WorkflowRunResult result = await workflow.RunAsync("Plan a trip to Paris");
|
||||
|
||||
IReadOnlyList<AgentEvaluationResults> evalResults = await result.EvaluateAsync(evals);
|
||||
|
||||
foreach (var r in evalResults)
|
||||
{
|
||||
Console.WriteLine($" overall: {r.Passed}/{r.Total}");
|
||||
foreach (var (name, sub) in r.SubResults)
|
||||
Console.WriteLine($" {name}: {sub.Passed}/{sub.Total}");
|
||||
}
|
||||
```
|
||||
|
||||
Workflows return one result per evaluator, with sub-results per agent in the workflow:
|
||||
|
||||
```
|
||||
EvalResults(status="completed", passed=2, failed=0, total=2)
|
||||
sub_results:
|
||||
"planner": EvalResults(passed=1, total=1)
|
||||
"researcher": EvalResults(passed=1, total=1)
|
||||
```
|
||||
|
||||
#### Mix multiple providers
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
@evaluator
|
||||
def is_helpful(response: str) -> bool:
|
||||
return len(response.split()) > 10
|
||||
|
||||
foundry = FoundryEvals(
|
||||
project_client=client,
|
||||
model_deployment="gpt-4o",
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=queries,
|
||||
evaluators=[is_helpful, keyword_check("weather"), foundry],
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
IReadOnlyList<AgentEvaluationResults> results = await agent.EvaluateAsync(
|
||||
queries,
|
||||
evaluators: new IAgentEvaluator[]
|
||||
{
|
||||
new LocalEvaluator(
|
||||
EvalChecks.KeywordCheck("weather"),
|
||||
FunctionEvaluator.Create("is_helpful", (string r) => r.Split(' ').Length > 10)),
|
||||
new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence),
|
||||
});
|
||||
```
|
||||
|
||||
Multiple evaluators return one result each — `results[0]` is the local evaluator, `results[1]` is Foundry.
|
||||
|
||||
#### Custom function evaluators
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
@evaluator
|
||||
def mentions_city(response: str, expected_output: str) -> bool:
|
||||
return expected_output.lower() in response.lower()
|
||||
|
||||
@evaluator
|
||||
def used_tools(conversation: list, tools: list) -> float:
|
||||
# ... scoring logic
|
||||
return score
|
||||
|
||||
local = LocalEvaluator(mentions_city, used_tools)
|
||||
```
|
||||
|
||||
`@evaluator` uses **parameter name injection** — the function's parameter names determine what data it receives from the `EvalItem`. Supported names: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`. Any combination is valid.
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var local = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("mentions_city",
|
||||
(EvalItem item) => item.ExpectedOutput != null
|
||||
&& item.Response.Contains(item.ExpectedOutput, StringComparison.OrdinalIgnoreCase)),
|
||||
FunctionEvaluator.Create("is_concise",
|
||||
(string response) => response.Split(' ').Length < 500));
|
||||
```
|
||||
|
||||
## What To Build
|
||||
|
||||
### Core: Evaluator Protocol
|
||||
|
||||
A runtime-checkable protocol that any evaluation provider implements:
|
||||
|
||||
```python
|
||||
@runtime_checkable
|
||||
class Evaluator(Protocol):
|
||||
name: str
|
||||
|
||||
async def evaluate(
|
||||
self, items: Sequence[EvalItem], *, eval_name: str = "Agent Framework Eval"
|
||||
) -> EvalResults: ...
|
||||
```
|
||||
|
||||
The protocol is minimal — just `name` and `evaluate()`.
|
||||
|
||||
### Core: EvalItem
|
||||
|
||||
Provider-agnostic data format for items to evaluate:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class ExpectedToolCall:
|
||||
name: str # Tool/function name
|
||||
arguments: dict[str, Any] | None = None # None = don't check args
|
||||
|
||||
@dataclass
|
||||
class EvalItem:
|
||||
conversation: list[Message] # Single source of truth
|
||||
tools: list[FunctionTool] | None = None # Agent's available tools
|
||||
context: str | None = None
|
||||
expected_output: str | None = None # Ground-truth for comparison
|
||||
expected_tool_calls: list[ExpectedToolCall] | None = None
|
||||
split_strategy: ConversationSplitter | None = None
|
||||
|
||||
query: str # property — derived from conversation split
|
||||
response: str # property — derived from conversation split
|
||||
```
|
||||
|
||||
`conversation` is the single source of truth. `query` and `response` are derived properties — splitting the conversation at the last user message (default) and extracting text from each side. Changing the `split_strategy` consistently changes all derived values.
|
||||
|
||||
`tools` provides typed `FunctionTool` objects — including MCP tools, which are automatically extracted after agent runs.
|
||||
|
||||
### Internal: AgentEvalConverter
|
||||
|
||||
Internal class that converts agent-framework types to `EvalItem`. Used by `evaluate_agent()` and `evaluate_workflow()` — not part of the public API:
|
||||
|
||||
| Agent Framework | Eval Format |
|
||||
|---|---|
|
||||
| `Content.function_call` | `tool_call` in OpenAI chat format |
|
||||
| `Content.function_result` | `tool_result` in OpenAI chat format |
|
||||
| `FunctionTool` | `{name, description, parameters}` schema |
|
||||
| `Message` history | `conversation` list + `query`/`response` extraction |
|
||||
|
||||
### Core: EvalResults
|
||||
|
||||
Rich result type with convenience properties for CI integration:
|
||||
|
||||
```python
|
||||
results.all_passed # bool: no failures or errors (recursive for workflow)
|
||||
results.passed # int: passing count
|
||||
results.failed # int: failure count
|
||||
results.total # int: total = passed + failed + errored
|
||||
results.items # list[EvalItemResult]: per-item detail with query, response, and scores
|
||||
results.error # str | None: error details on failure
|
||||
results.sub_results # dict: per-agent breakdown (workflow evals)
|
||||
results.report_url # str | None: portal link (Foundry)
|
||||
results.assert_passed() # raises AssertionError with details
|
||||
```
|
||||
|
||||
### Core: Orchestration Functions
|
||||
|
||||
Provider-agnostic functions that extract data and delegate to evaluators:
|
||||
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `evaluate_agent()` | Runs agent against test queries (or evaluates pre-existing `responses=`), converts to `EvalItem`s, passes to evaluator. Accepts optional `expected_output=` for ground-truth comparison, `expected_tool_calls=` for tool-correctness evaluation, and `num_repetitions=` for consistency measurement |
|
||||
| `evaluate_workflow()` | Extracts per-agent data from `WorkflowRunResult`, evaluates each agent and overall output. Per-agent breakdown in `sub_results`. Also accepts `num_repetitions=` |
|
||||
|
||||
### Core: Conversation Split Strategies
|
||||
|
||||
Multi-turn conversations must be split into query (input) and response (output) halves for evaluation. How you split determines *what you're evaluating*:
|
||||
|
||||
**Last-turn split** — split at the last user message. Everything up to and including it is the query context; the agent's subsequent actions are the response:
|
||||
|
||||
```
|
||||
conversation: user1 → assistant1 → user2 → assistant2(tool) → tool_result → assistant3
|
||||
query_messages: [user1, assistant1, user2]
|
||||
response_messages: [assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates: "Given all the context so far, did the agent answer the latest question well?" Best for response quality at a specific point in the conversation.
|
||||
|
||||
**Full-conversation split** — the first user message is the query; everything after is the response:
|
||||
|
||||
```
|
||||
query_messages: [user1]
|
||||
response_messages: [assistant1, user2, assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates: "Given the original request, did the entire conversation trajectory serve the user?" Best for task completion and overall conversation quality.
|
||||
|
||||
**Per-turn split** — produces N eval items from an N-turn conversation. Each turn is evaluated with its cumulative context:
|
||||
|
||||
```
|
||||
item 1: query = [user1], response = [assistant1]
|
||||
item 2: query = [user1, assistant1, user2], response = [assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates each response independently. Best for fine-grained analysis and pinpointing where a conversation goes wrong.
|
||||
|
||||
These factorings produce different scores for the same conversation. The framework ships all three as built-in strategies, defaulting to last-turn. Developers can also provide a custom splitter — a function (Python) or `IConversationSplitter` implementation (.NET) — and override the strategy at the call site or per evaluator.
|
||||
|
||||
### Azure AI: FoundryEvals
|
||||
|
||||
`Evaluator` implementation backed by Azure AI Foundry:
|
||||
|
||||
```python
|
||||
class FoundryEvals:
|
||||
def __init__(self, *, project_client=None, openai_client=None,
|
||||
model_deployment: str, evaluators=None, ...)
|
||||
async def evaluate(self, items, *, eval_name) -> EvalResults
|
||||
```
|
||||
|
||||
**Smart auto-detection in `evaluate()`:**
|
||||
- Default evaluators: relevance, coherence, task_adherence
|
||||
- Auto-adds `tool_call_accuracy` when items have tools/`tool_definitions`
|
||||
- Filters out tool evaluators for items without tools
|
||||
|
||||
### Azure AI: FoundryEvals Constants
|
||||
|
||||
```python
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
|
||||
evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY]
|
||||
```
|
||||
|
||||
Categories: Agent behavior, Tool usage, Quality, Safety.
|
||||
|
||||
### Azure AI: Foundry-Specific Functions
|
||||
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `evaluate_traces()` | Evaluate from stored response IDs or OTel traces |
|
||||
| `evaluate_foundry_target()` | Evaluate a Foundry-registered agent or deployment |
|
||||
|
||||
### Core: LocalEvaluator and Function Evaluators
|
||||
|
||||
`LocalEvaluator` implements the `Evaluator` protocol for fast, API-free evaluation. It runs check functions locally — useful for inner-loop development, CI smoke tests, and combining with cloud-based evaluators.
|
||||
|
||||
Built-in checks:
|
||||
- `keyword_check(*keywords)` — response must contain specified keywords
|
||||
- `tool_called_check(*tool_names)` — agent must have called specified tools
|
||||
- `tool_calls_present` — all `expected_tool_calls` names appear in conversation (unordered, extras OK)
|
||||
- `tool_call_args_match` — expected tool calls match on name + arguments (subset match on args)
|
||||
|
||||
Custom function evaluators use `@evaluator` to wrap plain Python functions. The function's **parameter names** determine what data it receives from the `EvalItem`:
|
||||
|
||||
```python
|
||||
from agent_framework import evaluator, LocalEvaluator
|
||||
|
||||
# Tier 1: Simple check — just query + response
|
||||
@evaluator
|
||||
def is_concise(response: str) -> bool:
|
||||
return len(response.split()) < 500
|
||||
|
||||
# Tier 2: Ground truth — compare against expected output
|
||||
@evaluator
|
||||
def mentions_city(response: str, expected_output: str) -> bool:
|
||||
return expected_output.lower() in response.lower()
|
||||
|
||||
# Tier 3: Full context — inspect conversation and tools
|
||||
@evaluator
|
||||
def used_tools(conversation: list, tools: list) -> float:
|
||||
# ... scoring logic
|
||||
return score
|
||||
|
||||
local = LocalEvaluator(is_concise, mentions_city, used_tools)
|
||||
```
|
||||
|
||||
Supported parameters: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`.
|
||||
Return types: `bool`, `float` (≥0.5 = pass), `dict` with `score` or `passed` key, or `CheckResult`.
|
||||
|
||||
Async functions are handled automatically — `@evaluator` detects `async def` and produces the right wrapper.
|
||||
|
||||
### Example: GAIA Benchmark
|
||||
|
||||
[GAIA](https://huggingface.co/gaia-benchmark) tests real-world multi-step tasks with known expected answers. Each task has a question and a ground-truth answer, with optional file attachments. The framework accommodates GAIA's knobs (difficulty levels, file inputs, multi-step tool use) through the existing `EvalItem` fields:
|
||||
|
||||
```python
|
||||
from datasets import load_dataset
|
||||
from agent_framework import evaluate_agent, evaluator, LocalEvaluator
|
||||
|
||||
gaia = load_dataset("gaia-benchmark/GAIA", "2023_level1", split="test")
|
||||
|
||||
@evaluator
|
||||
def exact_match(response: str, expected_output: str) -> bool:
|
||||
return expected_output.strip().lower() in response.strip().lower()
|
||||
|
||||
# Simple path — evaluate_agent handles running + expected_output stamping
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[task["Question"] for task in gaia],
|
||||
expected_output=[task["Final answer"] for task in gaia],
|
||||
evaluators=LocalEvaluator(exact_match),
|
||||
)
|
||||
```
|
||||
|
||||
### Package Location
|
||||
|
||||
- Core types and orchestration: `agent_framework._eval`, `agent_framework._local_eval` (Python), `Microsoft.Agents.AI` (.NET)
|
||||
- Foundry provider: `agent_framework_azure_ai._foundry_evals` (Python), `Microsoft.Agents.AI.AzureAI` (.NET)
|
||||
- Azure-AI re-exports core types for convenience (Python)
|
||||
|
||||
## Known Limitations
|
||||
|
||||
1. **Tool evaluators require query + agent**: Tool evaluators need tool definition schemas. When using these evaluators with `evaluate_agent(responses=...)`, provide `queries=` and pass an agent with tool definitions.
|
||||
2. **`model_deployment` always required**: Could potentially be inferred from the Foundry project configuration.
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. **Red teaming non-registered agents**: Requires Foundry API support for callback-based flows.
|
||||
2. **Datasets with expected outputs**: A dataset abstraction for pre-populating `expected_output` values across eval runs is a natural next step but not yet designed.
|
||||
3. **Multi-modal evaluation**: The `conversation` field on `EvalItem` already stores full `Message`/`Content` (Python) and `ChatMessage` (.NET) objects, which can represent multi-modal content (images, audio, structured data). Evaluators that accept the full `EvalItem` or `conversation` parameter can access this content today. However, the convenience shortcuts — `query`/`response` string projections and the `FunctionEvaluator` string overloads — are text-only. Multi-modal-aware evaluators should use the full-item path (`Func<EvalItem, CheckResult>` in .NET, `conversation: list` parameter in Python).
|
||||
|
||||
## .NET Implementation Design
|
||||
|
||||
### Key Difference: MEAI Ecosystem
|
||||
|
||||
Unlike Python, the .NET ecosystem already has `Microsoft.Extensions.AI.Evaluation` (v10.3.0) providing:
|
||||
|
||||
- `IEvaluator` — per-item evaluation of `(messages, chatResponse) → EvaluationResult`
|
||||
- `CompositeEvaluator` — combines multiple evaluators
|
||||
- Quality evaluators — `RelevanceEvaluator`, `CoherenceEvaluator`, `GroundednessEvaluator`
|
||||
- Safety evaluators — `ContentHarmEvaluator`, `ProtectedMaterialEvaluator`
|
||||
- Metric types — `NumericMetric`, `BooleanMetric`, `StringMetric`
|
||||
|
||||
The .NET integration uses MEAI's `IEvaluator` directly — no new evaluator interface. Our contribution is the **orchestration layer**: extension methods that run agents, extract data, call `IEvaluator` per item, and aggregate results.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Developer Code │
|
||||
│ agent.EvaluateAsync(queries, evaluator) │
|
||||
│ run.EvaluateAsync(evaluator) │
|
||||
└────────────────┬─────────────────────────────────────────────┘
|
||||
│
|
||||
┌────────────────▼─────────────────────────────────────────────┐
|
||||
│ Orchestration Layer (Microsoft.Agents.AI) │
|
||||
│ AgentEvaluationExtensions — runs agents, extracts data, │
|
||||
│ calls IEvaluator per item, aggregates into │
|
||||
│ AgentEvaluationResults │
|
||||
└────────────────┬─────────────────────────────────────────────┘
|
||||
│ IEvaluator (MEAI)
|
||||
│
|
||||
┌───────────┼────────────┐
|
||||
│ │ │
|
||||
┌───▼───-┐ ┌───▼────┐ ┌────▼──────────┐
|
||||
│ MEAI │ │ Local │ │ Foundry │
|
||||
│ Quality│ │ Checks │ │ (cloud batch) │
|
||||
│ Safety │ │ Lambdas│ │ │
|
||||
└────────┘ └────────┘ └───────────────┘
|
||||
```
|
||||
|
||||
All evaluators implement MEAI's `IEvaluator`. The orchestration layer doesn't need to know which kind — it calls `EvaluateAsync(messages, chatResponse)` per item on all of them. `FoundryEvals` handles batching internally (buffers items, submits once, returns per-item results).
|
||||
|
||||
### .NET Core Types
|
||||
|
||||
**No new evaluator interface.** Use MEAI's `IEvaluator` directly.
|
||||
|
||||
**`AgentEvaluationResults`** — The only new type. Aggregates per-item MEAI `EvaluationResult`s across a batch of queries:
|
||||
|
||||
```csharp
|
||||
public class AgentEvaluationResults
|
||||
{
|
||||
public string Provider { get; init; }
|
||||
public string? ReportUrl { get; init; }
|
||||
|
||||
// Per-item — standard MEAI EvaluationResult, unchanged
|
||||
public IReadOnlyList<EvaluationResult> Items { get; init; }
|
||||
|
||||
// Aggregate pass/fail derived from metric interpretations
|
||||
public int Passed { get; }
|
||||
public int Failed { get; }
|
||||
public int Total { get; }
|
||||
public bool AllPassed { get; }
|
||||
|
||||
// Workflow: per-agent breakdown
|
||||
public IReadOnlyDictionary<string, AgentEvaluationResults>? SubResults { get; init; }
|
||||
|
||||
public void AssertAllPassed(string? message = null);
|
||||
}
|
||||
```
|
||||
|
||||
### .NET Evaluator Implementations
|
||||
|
||||
All implement MEAI's `IEvaluator`:
|
||||
|
||||
**`LocalEvaluator`** — Runs lambda checks locally, returns `BooleanMetric` per check:
|
||||
|
||||
```csharp
|
||||
var local = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("is_concise",
|
||||
(string response) => response.Split().Length < 500),
|
||||
EvalChecks.KeywordCheck("weather"),
|
||||
EvalChecks.ToolCalledCheck("get_weather"));
|
||||
```
|
||||
|
||||
**MEAI evaluators** — Used directly, no adapter needed:
|
||||
|
||||
```csharp
|
||||
var quality = new CompositeEvaluator(
|
||||
new RelevanceEvaluator(),
|
||||
new CoherenceEvaluator());
|
||||
```
|
||||
|
||||
**`FoundryEvals`** — Implements `IEvaluator` but batches internally. On first call, buffers the item. On the last item (or when explicitly flushed), submits the batch to Foundry and distributes per-item results:
|
||||
|
||||
```csharp
|
||||
var foundry = new FoundryEvals(projectClient, "gpt-4o");
|
||||
```
|
||||
|
||||
### .NET Orchestration: Extension Methods
|
||||
|
||||
```csharp
|
||||
public static class AgentEvaluationExtensions
|
||||
{
|
||||
// Evaluate an agent against test queries
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate pre-existing responses (without re-running the agent)
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
AgentResponse responses,
|
||||
IEvaluator evaluator,
|
||||
IEnumerable<string>? queries = null,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate with multiple evaluators (one result per evaluator)
|
||||
public static Task<IReadOnlyList<AgentEvaluationResults>> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEnumerable<IEvaluator> evaluators,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate a workflow run with per-agent breakdown
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this Run run,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
bool includeOverall = true,
|
||||
bool includePerAgent = true,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
|
||||
```csharp
|
||||
// MEAI evaluators — just works
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new RelevanceEvaluator(),
|
||||
chatConfiguration: new ChatConfiguration(evalClient));
|
||||
|
||||
// Local checks
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new LocalEvaluator(
|
||||
EvalChecks.KeywordCheck("weather")));
|
||||
|
||||
// Foundry cloud
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
|
||||
// Evaluate existing response (without re-running the agent)
|
||||
var response = await agent.RunAsync("What's the weather?");
|
||||
var results = await agent.EvaluateAsync(
|
||||
responses: response,
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
|
||||
// Mixed — one result per evaluator
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluators: [
|
||||
new LocalEvaluator(EvalChecks.KeywordCheck("weather")),
|
||||
new RelevanceEvaluator(),
|
||||
new FoundryEvals(projectClient, "gpt-4o")
|
||||
],
|
||||
chatConfiguration: new ChatConfiguration(evalClient));
|
||||
|
||||
// Workflow with per-agent breakdown
|
||||
Run run = await workflowRunner.RunAsync(workflow, "Plan a trip");
|
||||
var results = await run.EvaluateAsync(
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
```
|
||||
|
||||
### .NET Function Evaluators
|
||||
|
||||
Typed factory overloads (C# equivalent of Python's `@evaluator`):
|
||||
|
||||
```csharp
|
||||
public static class FunctionEvaluator
|
||||
{
|
||||
public static EvalCheck Create(string name, Func<string, bool> check); // response only
|
||||
public static EvalCheck Create(string name, Func<string, string?, bool> check); // expectedOutput
|
||||
public static EvalCheck Create(string name, Func<EvalItem, bool> check); // full item
|
||||
public static EvalCheck Create(string name, Func<EvalItem, CheckResult> check); // full control
|
||||
public static EvalCheck Create(string name, Func<string, Task<bool>> check); // async
|
||||
}
|
||||
```
|
||||
|
||||
`EvalItem` is a lightweight record used only by `FunctionEvaluator` and `LocalEvaluator` to pass context to check functions. It is not part of the `IEvaluator` interface:
|
||||
|
||||
```csharp
|
||||
public record ExpectedToolCall(string Name, IReadOnlyDictionary<string, object>? Arguments = null);
|
||||
|
||||
public sealed class EvalItem
|
||||
{
|
||||
public EvalItem(string query, string response, IReadOnlyList<ChatMessage> conversation);
|
||||
|
||||
public string Query { get; }
|
||||
public string Response { get; }
|
||||
public IReadOnlyList<ChatMessage> Conversation { get; }
|
||||
public IReadOnlyList<AITool>? Tools { get; set; }
|
||||
public string? ExpectedOutput { get; set; }
|
||||
public IReadOnlyList<ExpectedToolCall>? ExpectedToolCalls { get; set; }
|
||||
public string? Context { get; set; }
|
||||
public IConversationSplitter? Splitter { get; set; }
|
||||
}
|
||||
```
|
||||
|
||||
### Workflow Data Extraction (.NET)
|
||||
|
||||
`run.EvaluateAsync()` walks `Run.OutgoingEvents` via LINQ:
|
||||
|
||||
1. Pair `ExecutorInvokedEvent` / `ExecutorCompletedEvent` by `ExecutorId`
|
||||
2. Extract `AgentResponseEvent` for per-agent `ChatResponse`
|
||||
3. Call `evaluator.EvaluateAsync()` per invocation
|
||||
4. Group by `ExecutorId` for per-agent `SubResults`
|
||||
5. Use final workflow output for overall eval
|
||||
|
||||
### .NET Package Structure
|
||||
|
||||
| Package | Contents |
|
||||
|---------|----------|
|
||||
| `Microsoft.Agents.AI` | `IAgentEvaluator`, `AgentEvaluationResults`, `LocalEvaluator`, `FunctionEvaluator`, `EvalChecks`, `EvalItem`, `ExpectedToolCall`, `AgentEvaluationExtensions` |
|
||||
| `Microsoft.Agents.AI.AzureAI` | `FoundryEvals` (provider + constants) |
|
||||
|
||||
### Python ↔ .NET Mapping
|
||||
|
||||
| Python | .NET |
|
||||
|--------|------|
|
||||
| `Evaluator` protocol | `IAgentEvaluator` (our interface; MEAI provides `IEvaluator` for per-item scoring) |
|
||||
| `EvalItem` dataclass | `EvalItem` class |
|
||||
| `EvalResults` | `AgentEvaluationResults` |
|
||||
| `EvalItemResult` / `EvalScoreResult` | MEAI `EvaluationResult` / `EvaluationMetric` (reused) |
|
||||
| `LocalEvaluator` | `LocalEvaluator` (implements `IAgentEvaluator`) |
|
||||
| `@evaluator` | `FunctionEvaluator.Create()` overloads |
|
||||
| `keyword_check()` / `tool_called_check()` | `EvalChecks.KeywordCheck()` / `EvalChecks.ToolCalledCheck()` |
|
||||
| `tool_calls_present` / `tool_call_args_match` | (custom `FunctionEvaluator` — same pattern) |
|
||||
| `ExpectedToolCall` dataclass | `ExpectedToolCall` record |
|
||||
| `FoundryEvals` | `FoundryEvals` (implements `IAgentEvaluator`, includes evaluator name constants) |
|
||||
| `evaluate_agent()` | `agent.EvaluateAsync(queries, evaluator)` extension method |
|
||||
| `evaluate_agent(responses=)` | `agent.EvaluateAsync(responses, evaluator)` extension method |
|
||||
| `evaluate_workflow()` | `run.EvaluateAsync()` extension method |
|
||||
|
||||
## More Information
|
||||
|
||||
- [Foundry Evals documentation](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-approach-gen-ai) — Azure AI Foundry evaluation overview
|
||||
@@ -34,11 +34,11 @@ Key changes:
|
||||
1. **New `agent-framework-openai` package** with dependencies on `agent-framework-core`, `openai`, and `packaging` only.
|
||||
2. **Class renames**: `OpenAIResponsesClient` → `OpenAIChatClient` (Responses API), `OpenAIChatClient` → `OpenAIChatCompletionClient` (Chat Completions API). Old names remain as deprecated aliases.
|
||||
3. **Deprecated classes**: `OpenAIAssistantsClient`, all `AzureOpenAI*Client` classes, `AzureAIClient`, `AzureAIAgentClient`, and `AzureAIProjectAgentProvider` are marked deprecated.
|
||||
4. **New `FoundryChatClient`** in azure-ai for Microsoft Foundry Responses API access, built on `RawFoundryChatClient(RawOpenAIChatClient)`.
|
||||
4. **New `FoundryChatClient`** in azure-ai for Azure AI Foundry Responses API access, built on `RawFoundryChatClient(RawOpenAIChatClient)`.
|
||||
5. **All deprecated `AzureOpenAI*` classes** consolidated into a single file (`_deprecated_azure_openai.py`) in the azure-ai package for clean future deletion.
|
||||
6. **Core's `agent_framework.openai` and `agent_framework.azure` namespaces** become lazy-loading gateways, preserving backward-compatible import paths while removing hard dependencies.
|
||||
7. **Unified `model` parameter** replaces `model_id` (OpenAI), `deployment_name` (Azure OpenAI), and `model_deployment_name` (Azure AI) across all client constructors. The term `model` is intentionally generic: it naturally maps to an OpenAI model name *and* to an Azure OpenAI deployment name, making it straightforward to use `OpenAIChatClient` with either OpenAI or Azure OpenAI backends (via `AsyncAzureOpenAI`). Environment variables are similarly unified (e.g., `OPENAI_MODEL` instead of separate `OPENAI_CHAT_MODEL_ID` / `OPENAI_CHAT_COMPLETION_MODEL_ID`).
|
||||
8. **`FoundryAgent`** replaces the pattern of `Agent(client=AzureAIClient(...))` for connecting to pre-configured agents in Microsoft Foundry (PromptAgents and HostedAgents). The underlying `RawFoundryAgentChatClient` is an implementation detail — most users interact only with `FoundryAgent`. `AzureAIAgentClient` is separately deprecated as it refers to the V1 Agents Service API. See below for design rationale.
|
||||
7. **Unified `model` parameter** replaces `model_id` (OpenAI), `deployment_name` (Azure OpenAI), and `model_deployment_name` (Azure AI) across all client constructors. The term `model` is intentionally generic: it naturally maps to an OpenAI model name *and* to an Azure OpenAI deployment name, making it straightforward to use `OpenAIChatClient` with either OpenAI or Azure OpenAI backends (via `AsyncAzureOpenAI`). Environment variables are similarly unified (e.g., `OPENAI_MODEL` instead of separate `OPENAI_RESPONSES_MODEL_ID` / `OPENAI_CHAT_MODEL_ID`).
|
||||
8. **`FoundryAgent`** replaces the pattern of `Agent(client=AzureAIClient(...))` for connecting to pre-configured agents in Azure AI Foundry (PromptAgents and HostedAgents). The underlying `RawFoundryAgentChatClient` is an implementation detail — most users interact only with `FoundryAgent`. `AzureAIAgentClient` is separately deprecated as it refers to the V1 Agents Service API. See below for design rationale.
|
||||
|
||||
### Foundry Agent Design: `FoundryAgentClient` vs `FoundryAgent`
|
||||
|
||||
|
||||
@@ -31,6 +31,8 @@ The persistence timing and `FunctionResultContent` trimming behaviors are interr
|
||||
|
||||
- **Per-run persistence**: When messages are batched and persisted at the end of the full run, trailing `FunctionResultContent` trimming becomes necessary to match the service's behavior. Without trimming, the stored history contains `FunctionResultContent` that the service would never have stored.
|
||||
|
||||
This means the trimming feature (introduced in [PR #4792](https://github.com/microsoft/agent-framework/pull/4792)) is primarily needed as a complement to per-run persistence. The `PersistChatHistoryAtEndOfRun` setting (introduced in [PR #4762](https://github.com/microsoft/agent-framework/pull/4762)) inverts the default so that per-service-call persistence is the standard behavior, and per-run persistence is opt-in.
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- **A. Consistency**: The default behavior of `ChatHistoryProvider` should produce stored history that closely matches what the underlying AI service would store, minimizing surprise when switching between framework-managed and service-managed chat history.
|
||||
@@ -41,30 +43,33 @@ The persistence timing and `FunctionResultContent` trimming behaviors are interr
|
||||
|
||||
## Considered Options
|
||||
|
||||
- Option 1: Per-run persistence with opt-in FRC (FunctionResultContent) trimming
|
||||
- Option 2: Opt-in per-service-call persistence (via `RequirePerServiceCallChatHistoryPersistence`)
|
||||
- Option 1: Default to per-run persistence with `FunctionResultContent` trimming (opt-in to per-service-call)
|
||||
- Option 2: Default to per-service-call persistence (opt-in to per-run)
|
||||
|
||||
## Pros and Cons of the Options
|
||||
|
||||
### Option 1: Per-run persistence with opt-in FRC trimming
|
||||
### Option 1: Default to per-run persistence with `FunctionResultContent` trimming
|
||||
|
||||
Keep the current default behavior of persisting chat history only at the end of the full agent run. Add `FunctionResultContent` trimming as an opt-in behavior to improve consistency with service storage.
|
||||
Keep the current default behavior of persisting chat history only at the end of the full agent run. Add `FunctionResultContent` trimming as the default to improve consistency with service storage. Provide an opt-in setting for users who want per-service-call persistence.
|
||||
|
||||
Settings:
|
||||
- `PersistChatHistoryAtEndOfRun` = `true`
|
||||
|
||||
- Good, because runs are atomic — chat history is only updated when the full run succeeds, satisfying driver B.
|
||||
- Good, because the mental model is simple: one run = one history update, satisfying driver D.
|
||||
- Good, because trimming trailing `FunctionResultContent` improves consistency with service storage, partially satisfying driver A.
|
||||
- Good, because users can opt in to per-service-call persistence for checkpointing/recovery scenarios, satisfying drivers C and E.
|
||||
- Bad, because the default persistence timing still differs from the service's behavior (per-run vs. per-service-call), only partially satisfying driver A.
|
||||
- Bad, because if the process crashes mid-loop, all intermediate progress from the current run is lost, not satisfying driver C.
|
||||
- Bad, because this option alone does not provide a way for users to opt into per-service-call persistence, not satisfying driver E.
|
||||
- Bad, because if the process crashes mid-loop, all intermediate progress from the current run is lost, not satisfying driver C by default.
|
||||
|
||||
### Option 2: Opt-in per-service-call persistence (via `RequirePerServiceCallChatHistoryPersistence`)
|
||||
### Option 2: Default to per-service-call persistence
|
||||
|
||||
Introduce an optional RequirePerServiceCallChatHistoryPersistence setting to persist chat history after each individual service call within the FIC loop, matching the AI service's behavior. Trailing `FunctionResultContent` trimming is unnecessary with this approach (it is naturally handled).
|
||||
Change the default to persist chat history after each individual service call within the FIC loop, matching the AI service's behavior. Trailing `FunctionResultContent` trimming is unnecessary with this approach (it is naturally handled). Provide an opt-in setting for users who want per-run atomicity with trimming.
|
||||
|
||||
Settings:
|
||||
- `RequirePerServiceCallChatHistoryPersistence` = `true`
|
||||
- `PersistChatHistoryAtEndOfRun` = `false` (default)
|
||||
|
||||
- Good, because the stored history matches the service's behavior when opting in for both timing and content, fully satisfying driver A.
|
||||
- Good, because the stored history matches the service's behavior by default for both timing and content, fully satisfying driver A.
|
||||
- Good, because intermediate progress is preserved if the process is interrupted, satisfying driver C.
|
||||
- Good, because no separate `FunctionResultContent` trimming logic is needed, reducing complexity.
|
||||
- Bad, because chat history may be left in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), not satisfying driver B. A subsequent run cannot proceed without manually providing the missing `FunctionResultContent`.
|
||||
@@ -73,49 +78,39 @@ Settings:
|
||||
|
||||
## Decision Outcome
|
||||
|
||||
Chosen option: **Option 2: Opt-in per-service-call persistence (via `RequirePerServiceCallChatHistoryPersistence`)**. The existing per-run persistence behavior is retained as-is, requiring no changes from users. Per-service-call persistence is available as an opt-in feature via the `RequirePerServiceCallChatHistoryPersistence` setting. This satisfies drivers B (atomicity) and D (simplicity) for the common case, while fully satisfying driver A (consistency) for users who opt into simulated service-stored behavior. Users who need per-service-call persistence for recoverability (driver C) can enable it explicitly.
|
||||
Chosen option: **Option 2 — Default to per-service-call persistence**, because it fully satisfies the consistency driver (A), naturally handles `FunctionResultContent` trimming without additional logic, and provides better recoverability for long-running tool-calling loops. Per-run persistence remains available via the `PersistChatHistoryAtEndOfRun` setting for users who prefer atomic run semantics.
|
||||
|
||||
### Configuration Matrix
|
||||
|
||||
The behavior depends on the combination of `UseProvidedChatClientAsIs` and `RequirePerServiceCallChatHistoryPersistence`:
|
||||
The behavior depends on the combination of `UseProvidedChatClientAsIs` and `PersistChatHistoryAtEndOfRun`:
|
||||
|
||||
| `UseProvidedChatClientAsIs` | `RequirePerServiceCallChatHistoryPersistence` | Behavior |
|
||||
| `UseProvidedChatClientAsIs` | `PersistChatHistoryAtEndOfRun` | Behavior |
|
||||
|---|---|---|
|
||||
| `false` (default) | `false` (default) | **Per-run persistence.** Messages are persisted at the end of the full agent run via the `ChatHistoryProvider`. |
|
||||
| `false` | `true` | **Per-service-call persistence (simulated).** A `PerServiceCallChatHistoryPersistingChatClient` middleware is automatically injected into the chat client pipeline between `FunctionInvokingChatClient` and the leaf `IChatClient`. Messages are persisted after each service call. A sentinel `ConversationId` causes FIC to treat the conversation as service-managed. |
|
||||
| `true` | `false` | **Per-run persistence.** No middleware is injected because the user has provided a custom chat client stack. Messages are persisted at the end of the run. |
|
||||
| `true` | `true` | **User responsibility.** The system checks whether the custom chat client stack includes a `PerServiceCallChatHistoryPersistingChatClient`. If not, a warning is emitted — the user is expected to have added their own per-service-call persistence mechanism. End-of-run persistence is skipped. |
|
||||
| `false` (default) | `false` (default) | **Per-service-call persistence.** A `ChatHistoryPersistingChatClient` middleware is automatically injected into the chat client pipeline between `FunctionInvokingChatClient` and the leaf `IChatClient`. Messages are persisted after each service call. |
|
||||
| `true` | `false` | **User responsibility.** No middleware is injected because the user has provided a custom chat client stack. The user is responsible for ensuring correct persistence behavior (e.g., by including their own persisting middleware). |
|
||||
| `false` | `true` | **Per-run persistence with marking.** A `ChatHistoryPersistingChatClient` middleware is injected, but configured to *mark* messages with metadata rather than store them immediately. At the end of the run, marked messages are stored. Trailing `FunctionResultContent` is trimmed. |
|
||||
| `true` | `true` | **Per-run persistence with warning.** The system checks whether the custom chat client stack includes a `ChatHistoryPersistingChatClient`. If not, a warning is emitted (particularly relevant for workflow handoff scenarios where trimming cannot be guaranteed). If no `ChatHistoryPersistingChatClient` is preset, all messages are stored at the end of the run, otherwise marked messages are stored. |
|
||||
|
||||
### Consequences
|
||||
|
||||
- Good, because per-run persistence is atomic by default — chat history is only updated when the full run succeeds, satisfying driver B.
|
||||
- Good, because the default mental model is simple: one run = one history update, satisfying driver D.
|
||||
- Good, because users who opt into `RequirePerServiceCallChatHistoryPersistence` get stored history that matches the service's behavior for both timing and content, fully satisfying driver A.
|
||||
- Good, because per-service-call persistence preserves intermediate progress if the process is interrupted, satisfying driver C when opted in.
|
||||
- Good, because no separate `FunctionResultContent` trimming logic is needed when per-service-call persistence is active — it is naturally handled.
|
||||
- Good, because conflict detection (configurable via `ThrowOnChatHistoryProviderConflict`, `WarnOnChatHistoryProviderConflict`, `ClearOnChatHistoryProviderConflict`) prevents misconfiguration when a service returns a `ConversationId` alongside a configured `ChatHistoryProvider`.
|
||||
- Bad, because per-service-call persistence (when opted in) may leave chat history in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), requiring manual recovery in rare cases.
|
||||
- Neutral, because users who want per-service-call consistency can opt in via `RequirePerServiceCallChatHistoryPersistence = true`, satisfying driver E.
|
||||
- Good, because the stored history matches the service's behavior by default for both timing and content, fully satisfying consistency (driver A).
|
||||
- Good, because intermediate progress is preserved if the process is interrupted, satisfying recoverability (driver C).
|
||||
- Good, because no separate `FunctionResultContent` trimming logic is needed in the default path, reducing complexity.
|
||||
- Good, because marking persisted messages with metadata enables deduplication and aids debugging.
|
||||
- Good, because warnings for custom chat client configurations without the persisting middleware help prevent silent failures in workflow handoff scenarios.
|
||||
- Bad, because chat history may be left in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), requiring manual recovery in rare cases.
|
||||
- Bad, because the mental model is more complex for the default path: a single run may produce multiple history updates.
|
||||
- Neutral, because users who prefer atomic run semantics can opt in to per-run persistence via `PersistChatHistoryAtEndOfRun = true`.
|
||||
- Neutral, because increased write frequency from per-service-call persistence may impact performance for some storage backends; this can be mitigated with a caching decorator.
|
||||
|
||||
### Implementation Notes
|
||||
|
||||
#### Conversation ID Consistency
|
||||
|
||||
When `RequirePerServiceCallChatHistoryPersistence` is enabled, the `PerServiceCallChatHistoryPersistingChatClient`
|
||||
decorator also updates `session.ConversationId` after each service call. This handles two scenarios:
|
||||
The `ChatHistoryPersistingChatClient` middleware must also update the session's `ConversationId` consistently for both response-based and conversation-based service interactions, ensuring the session always reflects the latest service-provided identifier.
|
||||
|
||||
1. **Framework-managed chat history** — the decorator sets a sentinel `ConversationId` on the response
|
||||
so that `FunctionInvokingChatClient` treats the conversation as service-managed (clearing accumulated
|
||||
history between iterations and not injecting duplicate `FunctionCallContent` during approval processing).
|
||||
|
||||
2. **Service-stored chat history** — when the service returns a real `ConversationId`, the decorator
|
||||
updates `session.ConversationId` immediately after each service call, rather than deferring the update
|
||||
to the end of the run. This ensures intermediate ConversationId changes are captured even if the
|
||||
process is interrupted mid-loop.
|
||||
|
||||
For some service-stored scenarios (e.g., the Conversations API with the Responses API), there is only
|
||||
one thread with one ID, so every service call returns the same ConversationId and this per-call update
|
||||
makes no practical difference. Enabling `RequirePerServiceCallChatHistoryPersistence` ensures consistent
|
||||
per-service-call behavior across all service types regardless of how they manage ConversationIds.
|
||||
## More Information
|
||||
|
||||
- [PR #4762: Persist messages during function call loop](https://github.com/microsoft/agent-framework/pull/4762) — introduces `PersistChatHistoryAfterEachServiceCall` option and `ChatHistoryPersistingChatClient` decorator
|
||||
- [PR #4792: Trim final FRC to match service storage](https://github.com/microsoft/agent-framework/pull/4792) — introduces `StoreFinalFunctionResultContent` option and `FilterFinalFunctionResultContent` logic
|
||||
- [Issue #2889](https://github.com/microsoft/agent-framework/issues/2889) — original issue tracking chat history persistence during function call loops
|
||||
|
||||
@@ -1,815 +0,0 @@
|
||||
---
|
||||
status: accepted
|
||||
contact: bentho
|
||||
date: 2026-02-27
|
||||
deciders: bentho, markwallace-microsoft, westey-m
|
||||
consulted: Pratyush Mishra, Shivam Shrivastava, Manni Arora (Centrica eval scenario)
|
||||
informed: Agent Framework team, Foundry Evals team
|
||||
---
|
||||
|
||||
# Agent Evaluation Architecture with Microsoft Foundry Integration
|
||||
|
||||
## Context and Problem Statement
|
||||
|
||||
Microsoft Foundry provides a rich evaluation service for AI agents — built-in evaluators for agent behavior (task adherence, intent resolution), tool usage (tool call accuracy, tool selection), quality (coherence, fluency, relevance), and safety (violence, self-harm, prohibited actions). Results are viewable in the Foundry portal with dashboards and comparison views.
|
||||
|
||||
However, using Foundry Evals with an agent-framework agent today requires significant manual effort. Developers must:
|
||||
|
||||
1. Transform agent-framework's `Message`/`Content` types into the OpenAI-style agent message schema that Foundry evaluators expect
|
||||
2. Map tool definitions from agent-framework's `FunctionTool` format to evaluator-compatible schemas
|
||||
3. Manually wire up the correct Foundry data source type (`azure_ai_traces`, `jsonl`, `azure_ai_target_completions`, etc.) depending on their scenario
|
||||
4. Handle App Insights trace ID queries, response ID collection, and eval polling
|
||||
|
||||
Additionally, evaluation is a concern that extends beyond any single provider. Developers may want to use local evaluators (LLM-as-judge, regex, keyword matching), third-party evaluation libraries, or multiple providers in combination. The architecture must support this without creating a Foundry-specific lock-in at the API level.
|
||||
|
||||
### Functional Requirements for Agent Evaluation
|
||||
|
||||
- **Single agents and workflows.** Evaluate both individual agent responses and multi-agent workflow results, with per-agent breakdown to pinpoint underperformance.
|
||||
- **One-shot and multi-turn conversations.** Capture full conversation trajectories — including tool calls and results — not just final query/response pairs.
|
||||
- **Conversation factoring.** Support splitting conversations into query/response in multiple ways (last turn, full trajectory, per-turn) because different factorings measure different things.
|
||||
- **Multiple providers, mix and match.** Run Foundry LLM-as-judge evaluators alongside fast local checks and custom evaluators on the same data, without restructuring code.
|
||||
- **Third-party extensibility.** Any evaluation library can participate by implementing the `Evaluator` protocol (Python) or `IAgentEvaluator` interface (.NET). No predetermined list of supported libraries — the protocol is intentionally simple (`evaluate(items) → results`) so that wrappers for libraries like DeepEval, RAGAS, or Promptfoo are straightforward to write.
|
||||
- **Bring your own evaluator.** Creating a custom evaluator should be as simple as writing a function.
|
||||
- **Evaluate without re-running.** Evaluate existing responses from logs or previous runs without invoking the agent again.
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- **Zero-friction evaluation**: Developers should go from "I have an agent" to "I have eval results" with minimal code.
|
||||
- **Provider-agnostic API**: Core evaluation capabilities must not be tied to any specific provider. Provider configuration should be separate from the evaluation call.
|
||||
- **Lowest concept count**: Introduce the fewest possible new types, abstractions, and APIs for developers to learn.
|
||||
- **Leverage existing knowledge**: The framework already knows which agents exist, what tools they have, and what conversations occurred. Evals should use this automatically rather than requiring the developer to re-specify it.
|
||||
- **Foundry-native results**: When using Foundry, results should be viewable in the Foundry portal with dashboards and comparison views.
|
||||
- **Progressive disclosure**: Simple scenarios should be near-zero code. Advanced scenarios should build on the same primitives.
|
||||
- **Cross-language parity**: Design must be implementable in both Python and .NET.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Provider-specific functions** — Build Foundry-specific helper functions (`evaluate_agent()`, etc.) directly in the Azure package. All eval functions take Foundry connection parameters.
|
||||
2. **Evaluator protocol with shared orchestration** — Define a provider-agnostic `Evaluator` protocol in the base agent library (`agent_framework` in Python, `Microsoft.Agents.AI` in .NET). Orchestration functions live alongside it. Providers implement the protocol.
|
||||
3. **Full eval framework** — Build comprehensive eval infrastructure including custom evaluator definitions, scoring profiles, and reporting inside agent-framework.
|
||||
|
||||
## Decision Outcome
|
||||
|
||||
Proposed option: "Evaluator protocol with shared orchestration", because it delivers the low-friction developer experience, supports multiple providers without API changes, and keeps the concept count low.
|
||||
|
||||
### Usage Examples
|
||||
|
||||
#### Evaluate an agent
|
||||
|
||||
The agent is invoked once per query by default. For statistically meaningful evaluation, provide multiple diverse queries. For measuring **consistency** (does the same query produce reliable results?), use `num_repetitions` to run each query N times independently:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
evals = FoundryEvals(
|
||||
project_client=client,
|
||||
model_deployment="gpt-4o",
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=my_agent,
|
||||
queries=[
|
||||
"What's the weather in Seattle?",
|
||||
"Plan a weekend trip to Portland",
|
||||
"What restaurants are near Pike Place?",
|
||||
],
|
||||
evaluators=evals,
|
||||
)
|
||||
for r in results:
|
||||
r.assert_passed()
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var evals = new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] {
|
||||
"What's the weather in Seattle?",
|
||||
"Plan a weekend trip to Portland",
|
||||
"What restaurants are near Pike Place?",
|
||||
},
|
||||
evals);
|
||||
|
||||
results.AssertAllPassed();
|
||||
```
|
||||
|
||||
`evaluate_agent` returns one `EvalResults` per evaluator. Each result contains per-item scores with the evaluated response for auditing:
|
||||
|
||||
```
|
||||
# results[0] (FoundryEvals)
|
||||
EvalResults(status="completed", passed=3, failed=0, total=3)
|
||||
items[0]: EvalItemResult(
|
||||
query="What's the weather in Seattle?",
|
||||
response="It's currently 72°F and sunny in Seattle.",
|
||||
scores={"relevance": 5, "coherence": 5})
|
||||
items[1]: EvalItemResult(
|
||||
query="Plan a weekend trip to Portland",
|
||||
response="Here's a 2-day Portland itinerary...",
|
||||
scores={"relevance": 4, "coherence": 5})
|
||||
items[2]: EvalItemResult(
|
||||
query="What restaurants are near Pike Place?",
|
||||
response="Top restaurants near Pike Place Market: ...",
|
||||
scores={"relevance": 5, "coherence": 4})
|
||||
```
|
||||
|
||||
#### Measure consistency with repetitions
|
||||
|
||||
Run each query multiple times to detect non-deterministic behavior:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
results = await evaluate_agent(
|
||||
agent=my_agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=evals,
|
||||
num_repetitions=3, # each query runs 3 times independently
|
||||
)
|
||||
# results contain 3 items (1 query × 3 repetitions)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] { "What's the weather in Seattle?" },
|
||||
evals,
|
||||
numRepetitions: 3); // each query runs 3 times independently
|
||||
// results contain 3 items (1 query × 3 repetitions)
|
||||
```
|
||||
|
||||
#### Evaluate a response you already have
|
||||
|
||||
When you already have agent responses, pass them directly to skip re-running the agent. Each query is paired with its corresponding response:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
queries = ["What's the weather?", "What's the capital of France?"]
|
||||
responses = [await agent.run([Message("user", [q])]) for q in queries]
|
||||
|
||||
results = await evaluate_agent(
|
||||
responses=responses,
|
||||
evaluators=evals,
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var queries = new[] { "What's the weather?" };
|
||||
var responses = new List<AgentResponse>();
|
||||
foreach (var q in queries)
|
||||
responses.Add(await agent.RunAsync(new[] { new ChatMessage(ChatRole.User, q) }));
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
responses: responses,
|
||||
evals);
|
||||
```
|
||||
|
||||
Each `AgentResponse` already contains the conversation (query + response), so the evaluator extracts query/response from the conversation. When you pass `responses` without `queries`, the conversation is the source of truth.
|
||||
|
||||
#### Evaluate with conversation split strategies
|
||||
|
||||
By default, evaluators see only the last turn (final user message → final assistant response). For multi-turn conversations, you can control how the conversation is factored for evaluation:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["Plan a 3-day trip to Paris"],
|
||||
evaluators=evals,
|
||||
conversation_split=ConversationSplit.FULL, # evaluate entire trajectory
|
||||
)
|
||||
|
||||
# Or per-turn: each user→assistant exchange scored independently
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["Plan a 3-day trip to Paris"],
|
||||
evaluators=evals,
|
||||
conversation_split=ConversationSplit.PER_TURN,
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
// Full conversation as context
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] { "Plan a 3-day trip to Paris" },
|
||||
evals,
|
||||
splitter: ConversationSplitters.Full);
|
||||
|
||||
// Per-turn splitting
|
||||
var items = EvalItem.PerTurnItems(conversation); // one EvalItem per user turn
|
||||
var results = await evals.EvaluateAsync(items);
|
||||
```
|
||||
|
||||
With `PER_TURN`, a 3-turn conversation produces 3 scored items:
|
||||
|
||||
```
|
||||
EvalResults(status="completed", passed=3, failed=0, total=3)
|
||||
items[0]: query="Plan a 3-day trip to Paris" scores={"relevance": 5}
|
||||
items[1]: query="What about restaurants?" scores={"relevance": 4}
|
||||
items[2]: query="Make it budget-friendly" scores={"relevance": 5}
|
||||
```
|
||||
|
||||
#### Evaluate a multi-agent workflow
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
result = await workflow.run("Plan a trip to Paris")
|
||||
eval_results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
workflow_result=result,
|
||||
evaluators=evals,
|
||||
)
|
||||
|
||||
for r in eval_results:
|
||||
print(f" overall: {r.passed}/{r.total}")
|
||||
for name, sub in r.sub_results.items():
|
||||
print(f" {name}: {sub.passed}/{sub.total}")
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
WorkflowRunResult result = await workflow.RunAsync("Plan a trip to Paris");
|
||||
|
||||
IReadOnlyList<AgentEvaluationResults> evalResults = await result.EvaluateAsync(evals);
|
||||
|
||||
foreach (var r in evalResults)
|
||||
{
|
||||
Console.WriteLine($" overall: {r.Passed}/{r.Total}");
|
||||
foreach (var (name, sub) in r.SubResults)
|
||||
Console.WriteLine($" {name}: {sub.Passed}/{sub.Total}");
|
||||
}
|
||||
```
|
||||
|
||||
Workflows return one result per evaluator, with sub-results per agent in the workflow:
|
||||
|
||||
```
|
||||
EvalResults(status="completed", passed=2, failed=0, total=2)
|
||||
sub_results:
|
||||
"planner": EvalResults(passed=1, total=1)
|
||||
"researcher": EvalResults(passed=1, total=1)
|
||||
```
|
||||
|
||||
#### Mix multiple providers
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
@evaluator
|
||||
def is_helpful(response: str) -> bool:
|
||||
return len(response.split()) > 10
|
||||
|
||||
foundry = FoundryEvals(
|
||||
project_client=client,
|
||||
model_deployment="gpt-4o",
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=queries,
|
||||
evaluators=[is_helpful, keyword_check("weather"), foundry],
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
IReadOnlyList<AgentEvaluationResults> results = await agent.EvaluateAsync(
|
||||
queries,
|
||||
evaluators: new IAgentEvaluator[]
|
||||
{
|
||||
new LocalEvaluator(
|
||||
EvalChecks.KeywordCheck("weather"),
|
||||
FunctionEvaluator.Create("is_helpful", (string r) => r.Split(' ').Length > 10)),
|
||||
new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence),
|
||||
});
|
||||
```
|
||||
|
||||
Multiple evaluators return one result each — `results[0]` is the local evaluator, `results[1]` is Foundry.
|
||||
|
||||
#### Custom function evaluators
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
@evaluator
|
||||
def mentions_city(response: str, expected_output: str) -> bool:
|
||||
return expected_output.lower() in response.lower()
|
||||
|
||||
@evaluator
|
||||
def used_tools(conversation: list, tools: list) -> float:
|
||||
# ... scoring logic
|
||||
return score
|
||||
|
||||
local = LocalEvaluator(mentions_city, used_tools)
|
||||
```
|
||||
|
||||
`@evaluator` uses **parameter name injection** — the function's parameter names determine what data it receives from the `EvalItem`. Supported names: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`. Any combination is valid.
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var local = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("mentions_city",
|
||||
(EvalItem item) => item.ExpectedOutput != null
|
||||
&& item.Response.Contains(item.ExpectedOutput, StringComparison.OrdinalIgnoreCase)),
|
||||
FunctionEvaluator.Create("is_concise",
|
||||
(string response) => response.Split(' ').Length < 500));
|
||||
```
|
||||
|
||||
## What To Build
|
||||
|
||||
### Core: Evaluator Protocol
|
||||
|
||||
A runtime-checkable protocol that any evaluation provider implements:
|
||||
|
||||
```python
|
||||
@runtime_checkable
|
||||
class Evaluator(Protocol):
|
||||
name: str
|
||||
|
||||
async def evaluate(
|
||||
self, items: Sequence[EvalItem], *, eval_name: str = "Agent Framework Eval"
|
||||
) -> EvalResults: ...
|
||||
```
|
||||
|
||||
The protocol is minimal — just `name` and `evaluate()`.
|
||||
|
||||
### Core: EvalItem
|
||||
|
||||
Provider-agnostic data format for items to evaluate:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class ExpectedToolCall:
|
||||
name: str # Tool/function name
|
||||
arguments: dict[str, Any] | None = None # None = don't check args
|
||||
|
||||
@dataclass
|
||||
class EvalItem:
|
||||
conversation: list[Message] # Single source of truth
|
||||
tools: list[FunctionTool] | None = None # Agent's available tools
|
||||
context: str | None = None
|
||||
expected_output: str | None = None # Ground-truth for comparison
|
||||
expected_tool_calls: list[ExpectedToolCall] | None = None
|
||||
split_strategy: ConversationSplitter | None = None
|
||||
|
||||
query: str # property — derived from conversation split
|
||||
response: str # property — derived from conversation split
|
||||
```
|
||||
|
||||
`conversation` is the single source of truth. `query` and `response` are derived properties — splitting the conversation at the last user message (default) and extracting text from each side. Changing the `split_strategy` consistently changes all derived values.
|
||||
|
||||
`tools` provides typed `FunctionTool` objects — including MCP tools, which are automatically extracted after agent runs.
|
||||
|
||||
### Internal: AgentEvalConverter
|
||||
|
||||
Internal class that converts agent-framework types to `EvalItem`. Used by `evaluate_agent()` and `evaluate_workflow()` — not part of the public API:
|
||||
|
||||
| Agent Framework | Eval Format |
|
||||
|---|---|
|
||||
| `Content.function_call` | `tool_call` in OpenAI chat format |
|
||||
| `Content.function_result` | `tool_result` in OpenAI chat format |
|
||||
| `FunctionTool` | `{name, description, parameters}` schema |
|
||||
| `Message` history | `conversation` list + `query`/`response` extraction |
|
||||
|
||||
### Core: EvalResults
|
||||
|
||||
Rich result type with convenience properties for CI integration:
|
||||
|
||||
```python
|
||||
results.all_passed # bool: no failures or errors (recursive for workflow)
|
||||
results.passed # int: passing count
|
||||
results.failed # int: failure count
|
||||
results.total # int: total = passed + failed + errored
|
||||
results.items # list[EvalItemResult]: per-item detail with query, response, and scores
|
||||
results.error # str | None: error details on failure
|
||||
results.sub_results # dict: per-agent breakdown (workflow evals)
|
||||
results.report_url # str | None: portal link (Foundry)
|
||||
results.assert_passed() # raises AssertionError with details
|
||||
```
|
||||
|
||||
### Core: Orchestration Functions
|
||||
|
||||
Provider-agnostic functions that extract data and delegate to evaluators:
|
||||
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `evaluate_agent()` | Runs agent against test queries (or evaluates pre-existing `responses=`), converts to `EvalItem`s, passes to evaluator. Accepts optional `expected_output=` for ground-truth comparison, `expected_tool_calls=` for tool-correctness evaluation, and `num_repetitions=` for consistency measurement |
|
||||
| `evaluate_workflow()` | Extracts per-agent data from `WorkflowRunResult`, evaluates each agent and overall output. Per-agent breakdown in `sub_results`. Also accepts `num_repetitions=` |
|
||||
|
||||
### Core: Conversation Split Strategies
|
||||
|
||||
Multi-turn conversations must be split into query (input) and response (output) halves for evaluation. How you split determines *what you're evaluating*:
|
||||
|
||||
**Last-turn split** — split at the last user message. Everything up to and including it is the query context; the agent's subsequent actions are the response:
|
||||
|
||||
```
|
||||
conversation: user1 → assistant1 → user2 → assistant2(tool) → tool_result → assistant3
|
||||
query_messages: [user1, assistant1, user2]
|
||||
response_messages: [assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates: "Given all the context so far, did the agent answer the latest question well?" Best for response quality at a specific point in the conversation.
|
||||
|
||||
**Full-conversation split** — the first user message is the query; everything after is the response:
|
||||
|
||||
```
|
||||
query_messages: [user1]
|
||||
response_messages: [assistant1, user2, assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates: "Given the original request, did the entire conversation trajectory serve the user?" Best for task completion and overall conversation quality.
|
||||
|
||||
**Per-turn split** — produces N eval items from an N-turn conversation. Each turn is evaluated with its cumulative context:
|
||||
|
||||
```
|
||||
item 1: query = [user1], response = [assistant1]
|
||||
item 2: query = [user1, assistant1, user2], response = [assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates each response independently. Best for fine-grained analysis and pinpointing where a conversation goes wrong.
|
||||
|
||||
These factorings produce different scores for the same conversation. The framework ships all three as built-in strategies, defaulting to last-turn. Developers can also provide a custom splitter — a function (Python) or `IConversationSplitter` implementation (.NET) — and override the strategy at the call site or per evaluator.
|
||||
|
||||
### Azure AI: FoundryEvals
|
||||
|
||||
`Evaluator` implementation backed by Microsoft Foundry:
|
||||
|
||||
```python
|
||||
class FoundryEvals:
|
||||
def __init__(self, *, project_client=None, openai_client=None,
|
||||
model_deployment: str, evaluators=None, ...)
|
||||
async def evaluate(self, items, *, eval_name) -> EvalResults
|
||||
```
|
||||
|
||||
**Smart auto-detection in `evaluate()`:**
|
||||
- Default evaluators: relevance, coherence, task_adherence
|
||||
- Auto-adds `tool_call_accuracy` when items have tools/`tool_definitions`
|
||||
- Filters out tool evaluators for items without tools
|
||||
|
||||
### Azure AI: FoundryEvals Constants
|
||||
|
||||
```python
|
||||
from agent_framework.foundry import FoundryEvals
|
||||
|
||||
evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY]
|
||||
```
|
||||
|
||||
Categories: Agent behavior, Tool usage, Quality, Safety.
|
||||
|
||||
### Azure AI: Foundry-Specific Functions
|
||||
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `evaluate_traces()` | Evaluate from stored response IDs or OTel traces |
|
||||
| `evaluate_foundry_target()` | Evaluate a Foundry-registered agent or deployment |
|
||||
|
||||
### Core: LocalEvaluator and Function Evaluators
|
||||
|
||||
`LocalEvaluator` implements the `Evaluator` protocol for fast, API-free evaluation. It runs check functions locally — useful for inner-loop development, CI smoke tests, and combining with cloud-based evaluators.
|
||||
|
||||
Built-in checks:
|
||||
- `keyword_check(*keywords)` — response must contain specified keywords
|
||||
- `tool_called_check(*tool_names)` — agent must have called specified tools
|
||||
- `tool_calls_present` — all `expected_tool_calls` names appear in conversation (unordered, extras OK)
|
||||
- `tool_call_args_match` — expected tool calls match on name + arguments (subset match on args)
|
||||
|
||||
Custom function evaluators use `@evaluator` to wrap plain Python functions. The function's **parameter names** determine what data it receives from the `EvalItem`:
|
||||
|
||||
```python
|
||||
from agent_framework import evaluator, LocalEvaluator
|
||||
|
||||
# Tier 1: Simple check — just query + response
|
||||
@evaluator
|
||||
def is_concise(response: str) -> bool:
|
||||
return len(response.split()) < 500
|
||||
|
||||
# Tier 2: Ground truth — compare against expected output
|
||||
@evaluator
|
||||
def mentions_city(response: str, expected_output: str) -> bool:
|
||||
return expected_output.lower() in response.lower()
|
||||
|
||||
# Tier 3: Full context — inspect conversation and tools
|
||||
@evaluator
|
||||
def used_tools(conversation: list, tools: list) -> float:
|
||||
# ... scoring logic
|
||||
return score
|
||||
|
||||
local = LocalEvaluator(is_concise, mentions_city, used_tools)
|
||||
```
|
||||
|
||||
Supported parameters: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`.
|
||||
Return types: `bool`, `float` (≥0.5 = pass), `dict` with `score` or `passed` key, or `CheckResult`.
|
||||
|
||||
Async functions are handled automatically — `@evaluator` detects `async def` and produces the right wrapper.
|
||||
|
||||
### Example: GAIA Benchmark
|
||||
|
||||
[GAIA](https://huggingface.co/gaia-benchmark) tests real-world multi-step tasks with known expected answers. Each task has a question and a ground-truth answer, with optional file attachments. The framework accommodates GAIA's knobs (difficulty levels, file inputs, multi-step tool use) through the existing `EvalItem` fields:
|
||||
|
||||
```python
|
||||
from datasets import load_dataset
|
||||
from agent_framework import evaluate_agent, evaluator, LocalEvaluator
|
||||
|
||||
gaia = load_dataset("gaia-benchmark/GAIA", "2023_level1", split="test")
|
||||
|
||||
@evaluator
|
||||
def exact_match(response: str, expected_output: str) -> bool:
|
||||
return expected_output.strip().lower() in response.strip().lower()
|
||||
|
||||
# Simple path — evaluate_agent handles running + expected_output stamping
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[task["Question"] for task in gaia],
|
||||
expected_output=[task["Final answer"] for task in gaia],
|
||||
evaluators=LocalEvaluator(exact_match),
|
||||
)
|
||||
```
|
||||
|
||||
### Package Location
|
||||
|
||||
- Core types and orchestration: `agent_framework._eval`, `agent_framework._local_eval` (Python), `Microsoft.Agents.AI` (.NET)
|
||||
- Foundry provider: `agent_framework_azure_ai._foundry_evals` (Python), `Microsoft.Agents.AI.AzureAI` (.NET)
|
||||
- Azure-AI re-exports core types for convenience (Python)
|
||||
|
||||
## Known Limitations
|
||||
|
||||
1. **Tool evaluators require query + agent**: Tool evaluators need tool definition schemas. When using these evaluators with `evaluate_agent(responses=...)`, provide `queries=` and pass an agent with tool definitions.
|
||||
2. **`model_deployment` always required**: Could potentially be inferred from the Foundry project configuration.
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. **Red teaming non-registered agents**: Requires Foundry API support for callback-based flows.
|
||||
2. **Datasets with expected outputs**: A dataset abstraction for pre-populating `expected_output` values across eval runs is a natural next step but not yet designed.
|
||||
3. **Multi-modal evaluation**: The `conversation` field on `EvalItem` already stores full `Message`/`Content` (Python) and `ChatMessage` (.NET) objects, which can represent multi-modal content (images, audio, structured data). Evaluators that accept the full `EvalItem` or `conversation` parameter can access this content today. However, the convenience shortcuts — `query`/`response` string projections and the `FunctionEvaluator` string overloads — are text-only. Multi-modal-aware evaluators should use the full-item path (`Func<EvalItem, CheckResult>` in .NET, `conversation: list` parameter in Python).
|
||||
|
||||
## .NET Implementation Design
|
||||
|
||||
### Key Difference: MEAI Ecosystem
|
||||
|
||||
Unlike Python, the .NET ecosystem already has `Microsoft.Extensions.AI.Evaluation` (v10.3.0) providing:
|
||||
|
||||
- `IEvaluator` — per-item evaluation of `(messages, chatResponse) → EvaluationResult`
|
||||
- `CompositeEvaluator` — combines multiple evaluators
|
||||
- Quality evaluators — `RelevanceEvaluator`, `CoherenceEvaluator`, `GroundednessEvaluator`
|
||||
- Safety evaluators — `ContentHarmEvaluator`, `ProtectedMaterialEvaluator`
|
||||
- Metric types — `NumericMetric`, `BooleanMetric`, `StringMetric`
|
||||
|
||||
The .NET integration uses MEAI's `IEvaluator` directly — no new evaluator interface. Our contribution is the **orchestration layer**: extension methods that run agents, extract data, call `IEvaluator` per item, and aggregate results.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Developer Code │
|
||||
│ agent.EvaluateAsync(queries, evaluator) │
|
||||
│ run.EvaluateAsync(evaluator) │
|
||||
└────────────────┬─────────────────────────────────────────────┘
|
||||
│
|
||||
┌────────────────▼─────────────────────────────────────────────┐
|
||||
│ Orchestration Layer (Microsoft.Agents.AI) │
|
||||
│ AgentEvaluationExtensions — runs agents, extracts data, │
|
||||
│ calls IEvaluator per item, aggregates into │
|
||||
│ AgentEvaluationResults │
|
||||
└────────────────┬─────────────────────────────────────────────┘
|
||||
│ IEvaluator (MEAI)
|
||||
│
|
||||
┌───────────┼────────────┐
|
||||
│ │ │
|
||||
┌───▼───-┐ ┌───▼────┐ ┌────▼──────────┐
|
||||
│ MEAI │ │ Local │ │ Foundry │
|
||||
│ Quality│ │ Checks │ │ (cloud batch) │
|
||||
│ Safety │ │ Lambdas│ │ │
|
||||
└────────┘ └────────┘ └───────────────┘
|
||||
```
|
||||
|
||||
All evaluators implement MEAI's `IEvaluator`. The orchestration layer doesn't need to know which kind — it calls `EvaluateAsync(messages, chatResponse)` per item on all of them. `FoundryEvals` handles batching internally (buffers items, submits once, returns per-item results).
|
||||
|
||||
### .NET Core Types
|
||||
|
||||
**No new evaluator interface.** Use MEAI's `IEvaluator` directly.
|
||||
|
||||
**`AgentEvaluationResults`** — The only new type. Aggregates per-item MEAI `EvaluationResult`s across a batch of queries:
|
||||
|
||||
```csharp
|
||||
public class AgentEvaluationResults
|
||||
{
|
||||
public string Provider { get; init; }
|
||||
public string? ReportUrl { get; init; }
|
||||
|
||||
// Per-item — standard MEAI EvaluationResult, unchanged
|
||||
public IReadOnlyList<EvaluationResult> Items { get; init; }
|
||||
|
||||
// Aggregate pass/fail derived from metric interpretations
|
||||
public int Passed { get; }
|
||||
public int Failed { get; }
|
||||
public int Total { get; }
|
||||
public bool AllPassed { get; }
|
||||
|
||||
// Workflow: per-agent breakdown
|
||||
public IReadOnlyDictionary<string, AgentEvaluationResults>? SubResults { get; init; }
|
||||
|
||||
public void AssertAllPassed(string? message = null);
|
||||
}
|
||||
```
|
||||
|
||||
### .NET Evaluator Implementations
|
||||
|
||||
All implement MEAI's `IEvaluator`:
|
||||
|
||||
**`LocalEvaluator`** — Runs lambda checks locally, returns `BooleanMetric` per check:
|
||||
|
||||
```csharp
|
||||
var local = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("is_concise",
|
||||
(string response) => response.Split().Length < 500),
|
||||
EvalChecks.KeywordCheck("weather"),
|
||||
EvalChecks.ToolCalledCheck("get_weather"));
|
||||
```
|
||||
|
||||
**MEAI evaluators** — Used directly, no adapter needed:
|
||||
|
||||
```csharp
|
||||
var quality = new CompositeEvaluator(
|
||||
new RelevanceEvaluator(),
|
||||
new CoherenceEvaluator());
|
||||
```
|
||||
|
||||
**`FoundryEvals`** — Implements `IEvaluator` but batches internally. On first call, buffers the item. On the last item (or when explicitly flushed), submits the batch to Foundry and distributes per-item results:
|
||||
|
||||
```csharp
|
||||
var foundry = new FoundryEvals(projectClient, "gpt-4o");
|
||||
```
|
||||
|
||||
### .NET Orchestration: Extension Methods
|
||||
|
||||
```csharp
|
||||
public static class AgentEvaluationExtensions
|
||||
{
|
||||
// Evaluate an agent against test queries
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate pre-existing responses (without re-running the agent)
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
AgentResponse responses,
|
||||
IEvaluator evaluator,
|
||||
IEnumerable<string>? queries = null,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate with multiple evaluators (one result per evaluator)
|
||||
public static Task<IReadOnlyList<AgentEvaluationResults>> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEnumerable<IEvaluator> evaluators,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate a workflow run with per-agent breakdown
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this Run run,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
bool includeOverall = true,
|
||||
bool includePerAgent = true,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
|
||||
```csharp
|
||||
// MEAI evaluators — just works
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new RelevanceEvaluator(),
|
||||
chatConfiguration: new ChatConfiguration(evalClient));
|
||||
|
||||
// Local checks
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new LocalEvaluator(
|
||||
EvalChecks.KeywordCheck("weather")));
|
||||
|
||||
// Foundry cloud
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
|
||||
// Evaluate existing response (without re-running the agent)
|
||||
var response = await agent.RunAsync("What's the weather?");
|
||||
var results = await agent.EvaluateAsync(
|
||||
responses: response,
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
|
||||
// Mixed — one result per evaluator
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluators: [
|
||||
new LocalEvaluator(EvalChecks.KeywordCheck("weather")),
|
||||
new RelevanceEvaluator(),
|
||||
new FoundryEvals(projectClient, "gpt-4o")
|
||||
],
|
||||
chatConfiguration: new ChatConfiguration(evalClient));
|
||||
|
||||
// Workflow with per-agent breakdown
|
||||
Run run = await workflowRunner.RunAsync(workflow, "Plan a trip");
|
||||
var results = await run.EvaluateAsync(
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
```
|
||||
|
||||
### .NET Function Evaluators
|
||||
|
||||
Typed factory overloads (C# equivalent of Python's `@evaluator`):
|
||||
|
||||
```csharp
|
||||
public static class FunctionEvaluator
|
||||
{
|
||||
public static EvalCheck Create(string name, Func<string, bool> check); // response only
|
||||
public static EvalCheck Create(string name, Func<string, string?, bool> check); // expectedOutput
|
||||
public static EvalCheck Create(string name, Func<EvalItem, bool> check); // full item
|
||||
public static EvalCheck Create(string name, Func<EvalItem, CheckResult> check); // full control
|
||||
public static EvalCheck Create(string name, Func<string, Task<bool>> check); // async
|
||||
}
|
||||
```
|
||||
|
||||
`EvalItem` is a lightweight record used only by `FunctionEvaluator` and `LocalEvaluator` to pass context to check functions. It is not part of the `IEvaluator` interface:
|
||||
|
||||
```csharp
|
||||
public record ExpectedToolCall(string Name, IReadOnlyDictionary<string, object>? Arguments = null);
|
||||
|
||||
public sealed class EvalItem
|
||||
{
|
||||
public EvalItem(string query, string response, IReadOnlyList<ChatMessage> conversation);
|
||||
|
||||
public string Query { get; }
|
||||
public string Response { get; }
|
||||
public IReadOnlyList<ChatMessage> Conversation { get; }
|
||||
public IReadOnlyList<AITool>? Tools { get; set; }
|
||||
public string? ExpectedOutput { get; set; }
|
||||
public IReadOnlyList<ExpectedToolCall>? ExpectedToolCalls { get; set; }
|
||||
public string? Context { get; set; }
|
||||
public IConversationSplitter? Splitter { get; set; }
|
||||
}
|
||||
```
|
||||
|
||||
### Workflow Data Extraction (.NET)
|
||||
|
||||
`run.EvaluateAsync()` walks `Run.OutgoingEvents` via LINQ:
|
||||
|
||||
1. Pair `ExecutorInvokedEvent` / `ExecutorCompletedEvent` by `ExecutorId`
|
||||
2. Extract `AgentResponseEvent` for per-agent `ChatResponse`
|
||||
3. Call `evaluator.EvaluateAsync()` per invocation
|
||||
4. Group by `ExecutorId` for per-agent `SubResults`
|
||||
5. Use final workflow output for overall eval
|
||||
|
||||
### .NET Package Structure
|
||||
|
||||
| Package | Contents |
|
||||
|---------|----------|
|
||||
| `Microsoft.Agents.AI` | `IAgentEvaluator`, `AgentEvaluationResults`, `LocalEvaluator`, `FunctionEvaluator`, `EvalChecks`, `EvalItem`, `ExpectedToolCall`, `AgentEvaluationExtensions` |
|
||||
| `Microsoft.Agents.AI.AzureAI` | `FoundryEvals` (provider + constants) |
|
||||
|
||||
### Python ↔ .NET Mapping
|
||||
|
||||
| Python | .NET |
|
||||
|--------|------|
|
||||
| `Evaluator` protocol | `IAgentEvaluator` (our interface; MEAI provides `IEvaluator` for per-item scoring) |
|
||||
| `EvalItem` dataclass | `EvalItem` class |
|
||||
| `EvalResults` | `AgentEvaluationResults` |
|
||||
| `EvalItemResult` / `EvalScoreResult` | MEAI `EvaluationResult` / `EvaluationMetric` (reused) |
|
||||
| `LocalEvaluator` | `LocalEvaluator` (implements `IAgentEvaluator`) |
|
||||
| `@evaluator` | `FunctionEvaluator.Create()` overloads |
|
||||
| `keyword_check()` / `tool_called_check()` | `EvalChecks.KeywordCheck()` / `EvalChecks.ToolCalledCheck()` |
|
||||
| `tool_calls_present` / `tool_call_args_match` | (custom `FunctionEvaluator` — same pattern) |
|
||||
| `ExpectedToolCall` dataclass | `ExpectedToolCall` record |
|
||||
| `FoundryEvals` | `FoundryEvals` (implements `IAgentEvaluator`, includes evaluator name constants) |
|
||||
| `evaluate_agent()` | `agent.EvaluateAsync(queries, evaluator)` extension method |
|
||||
| `evaluate_agent(responses=)` | `agent.EvaluateAsync(responses, evaluator)` extension method |
|
||||
| `evaluate_workflow()` | `run.EvaluateAsync()` extension method |
|
||||
|
||||
## More Information
|
||||
|
||||
- [Foundry Evals documentation](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-approach-gen-ai) — Microsoft Foundry evaluation overview
|
||||
@@ -1,233 +0,0 @@
|
||||
---
|
||||
status: proposed
|
||||
contact: eavanvalkenburg
|
||||
date: 2026-04-07
|
||||
deciders: TBD
|
||||
consulted:
|
||||
informed:
|
||||
---
|
||||
|
||||
# CodeAct integration through backend-specific context providers and an `execute_code` tool
|
||||
|
||||
## Introduction
|
||||
|
||||
**CodeAct** is a pattern in which the model writes executable code — rather than emitting a fixed function-call JSON schema — to plan, transform data, and orchestrate tool calls inside a single sandbox invocation. Instead of requiring a separate model round-trip for every tool call, conditional branch, or data transformation, the model produces a short program that runs in a controlled runtime, calls host-provided tools through a `call_tool(...)` bridge, and returns structured results. This reduces latency, lowers token cost, and lets the model express richer multi-step logic that is difficult to capture in a flat tool-call sequence.
|
||||
|
||||
Throughout this ADR, **CodeAct** is the primary term. **Code mode** and **programmatic tool calling** refer to the same capability.
|
||||
|
||||
## Context and Problem Statement
|
||||
|
||||
We need an architecture design that supports CodeAct in both Python and .NET. This is a necessary capability for the current generation of long-running agents, which need to plan, iterate, transform tool outputs, and execute bounded code inside a controlled runtime — for example, filtering a large result set, computing derived values, or chaining several tool calls with conditional logic — instead of requiring a separate model round-trip for each of those steps. The design should preserve the same behavioral contract across SDKs, but it does not need to use the same internal extension point in each runtime. We also want to standardize on Hyperlight as the initial backend, using the existing Python package and an anticipated .NET binding package once it is available.
|
||||
|
||||
Throughout this ADR, **CodeAct** is the primary term. **Code mode** and **programmatic tool calling** refer to the same capability. This ADR uses **CodeAct** consistently.
|
||||
|
||||
Model-generated code is treated as untrusted relative to the host process. This ADR assumes the selected backend provides the primary isolation boundary, while the framework is responsible for configuring approvals and capabilities, integrating telemetry, and translating outputs and failures into framework-native shapes. If a backend cannot provide isolation appropriate for its trust model, it is not a suitable CodeAct backend.
|
||||
|
||||
The core design question is: **where should CodeAct integrate into the agent pipeline so that both SDKs can offer the same functionality without invasive changes to their core function-calling loops?**
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- CodeAct must shape the model-facing surface before model invocation, not only after the model has already chosen tools.
|
||||
- The design should let users control which tools are available through CodeAct and which remain regular tools only.
|
||||
- The design must preserve existing session, approval, telemetry, and tool invocation behavior as much as possible.
|
||||
- The design should define the minimum cross-SDK telemetry and failure semantics for `execute_code`, so Python and .NET do not diverge on basic observability or error handling.
|
||||
- The design must fit naturally into the extension points that already exist in each SDK.
|
||||
- The design must be safe for concurrent runs and must not rely on mutating shared agent configuration during invocation.
|
||||
- The chosen structure should allow multiple backend-specific providers to fit under the same conceptual design over time, even though Hyperlight is the initial target.
|
||||
- The abstraction should not assume that every backend is a VM-style sandbox; alternative execution models such as Pydantic's Monty should also fit.
|
||||
- The design should allow `execute_code` to be reused both as a tool-enabled CodeAct runtime and as a standard code interpreter tool implementation.
|
||||
- The design should remain open to alternative language/runtime modes, such as JavaScript on Hyperlight, rather than baking the abstraction to Python only.
|
||||
- The design should provide a portable way to configure sandbox capabilities such as file access and network access, including allow-listed outbound domains.
|
||||
- Using CodeAct should be optional, and installing its runtime or backend dependencies should also be optional.
|
||||
- Backend-specific dependencies should be isolated behind a small adapter so SDK code is not tightly coupled to an unstable package surface.
|
||||
|
||||
## Considered Options
|
||||
|
||||
- **Option 1**: Standardize on context provider-based CodeAct with a shared cross-SDK contract and backend-specific public types
|
||||
- **Option 2**: Implement CodeAct as a dedicated chat-client decorator/wrapper
|
||||
- **Option 3**: Integrate CodeAct directly into the function invocation layer/FunctionInvokingChatClient
|
||||
|
||||
## Pros and Cons of the Options
|
||||
|
||||
### Option 1: Standardize on context provider-based CodeAct with a shared cross-SDK contract and backend-specific public types
|
||||
|
||||
This option uses `ContextProvider` in Python and `AIContextProvider` in .NET, but standardizes the public concept and behavior.
|
||||
In this option, the CodeAct tool set is provider-owned: only tools explicitly configured on the concrete CodeAct provider instance are available inside CodeAct, and the provider exposes direct CRUD-style management for tools, file mounts, and outbound network allow-list configuration rather than requiring a separate runtime setup object.
|
||||
The agent's direct tool surface remains separate. If a tool should be available both through CodeAct and as a normal direct tool, it is configured in both places.
|
||||
|
||||
- Good, because both SDKs already have first-class provider concepts intended for per-invocation context shaping.
|
||||
- Good, because providers operate before model invocation, which is where CodeAct must add instructions and reshape tools.
|
||||
- Good, because this lets us preserve existing function invocation behavior rather than rewriting it.
|
||||
- Good, because slightly different internals are acceptable while the public behavior remains aligned.
|
||||
- Good, because convenience builder/decorator helpers can still be added later on top of the provider model without changing the core design.
|
||||
- Good, because backend-specific runtime logic can stay inside concrete provider implementations or internal helpers instead of being forced into a lowest-common-denominator public abstraction.
|
||||
- Good, because the same provider structure can support either an all-or-nothing tool surface or a mixed side-by-side tool surface.
|
||||
- Good, because users can keep some tools direct-only while allowing other tools to be used from inside CodeAct.
|
||||
- Good, because a provider-owned CodeAct tool registry avoids mutating or inferring the agent's direct tool surface and can work consistently in both SDKs.
|
||||
- Good, because the same conceptual design can remain open to `HyperlightCodeActProvider`, a future `MontyCodeActProvider`, and other backend-specific providers over time.
|
||||
- Good, because `execute_code` can evolve into multiple backend-specific runtime modes rather than being hard-wired to one Python-plus-tools mode.
|
||||
- Bad, because the provider indirection adds per-run overhead — snapshotting the tool registry, dispatching lifecycle hooks, and building instructions — that a deeper integration point could skip. In practice this overhead is negligible relative to model inference latency and sandbox startup cost.
|
||||
|
||||
### Option 2: Implement CodeAct as a dedicated chat-client decorator/wrapper
|
||||
|
||||
This option would introduce a CodeAct-specific chat-client decorator that injects instructions and tools directly into the chat request pipeline.
|
||||
|
||||
- Good, because this is a natural fit for .NET's `DelegatingChatClient` pipeline.
|
||||
- Good, because it can also support advanced custom chat-client stacks.
|
||||
- Good, because backend-specific runtime selection could be hidden inside the decorator implementation.
|
||||
- Good, because the decorator could also encapsulate mode-specific instruction shaping for tool-enabled versus standalone interpreter behavior.
|
||||
- Good, because the decorator can decide per request whether the tool surface is exclusive or mixed.
|
||||
- Bad, because Python can support this by building a custom layering stack on top of a `Raw...Client` and swapping in a different `FunctionInvocationLayer`, but that composition path is more manual than the .NET `DelegatingChatClient` pipeline.
|
||||
- Bad, because it duplicates responsibilities already handled by provider abstractions.
|
||||
- Bad, because it makes CodeAct look more transport-specific than it really is.
|
||||
- Bad, because swappable backends and reusable interpreter or language modes become coupled to chat-client composition rather than modeled as first-class CodeAct concepts.
|
||||
|
||||
### Option 3: Integrate CodeAct directly into the function invocation layer/FunctionInvokingChatClient
|
||||
|
||||
This option would push CodeAct into Python's `FunctionInvocationLayer` and .NET's `FunctionInvokingChatClient` or related middleware.
|
||||
|
||||
- Good, because it is close to tool execution and can observe concrete tool invocation behavior.
|
||||
- Good, because function middleware may still be useful later for auxiliary auditing or policy around sandbox-originated tool calls.
|
||||
- Bad, because this is the wrong layer for constructing the model-facing tool surface and prompt instructions.
|
||||
- Bad, because it does not naturally control whether the model sees an exclusive CodeAct tool surface or a mixed side-by-side tool surface.
|
||||
- Bad, because it would still require a second mechanism for hiding normal tools and advertising `execute_code`.
|
||||
- Bad, because it is a weak fit for standalone interpreter modes where no tool-calling loop is needed.
|
||||
- Bad, because backend selection and CodeAct mode behavior are orthogonal concerns that do not belong in the function invocation layer.
|
||||
- Bad, because `.NET` would become more tightly coupled to `FunctionInvokingChatClient`, which sits below the agent framework abstraction and is not the natural cross-SDK design seam.
|
||||
|
||||
## Approval Model Options
|
||||
|
||||
- **Option A**: Bundled approval for the `execute_code` invocation
|
||||
- **Option B**: Pre-execution inspection of `call_tool(...)` references before approving `execute_code`
|
||||
- **Option C**: Nested per-tool approvals during `execute_code`
|
||||
|
||||
## Pros and Cons of the Approval Options
|
||||
|
||||
### Option A: Bundled approval for the `execute_code` invocation
|
||||
|
||||
This option grants approval once, before `execute_code` starts. Provider-owned tool calls made from inside that execution run under the same approval. The effective approval of `execute_code` is determined up front from the provider configuration rather than from inspecting which tools are actually called during execution.
|
||||
|
||||
- Good, because it is the simplest model to explain and implement consistently in both SDKs.
|
||||
- Good, because it fits naturally with long-running CodeAct loops where repeated approval interruptions would be disruptive.
|
||||
- Good, because it does not require static code analysis before execution begins.
|
||||
- Good, because it keeps the first release focused on the provider integration rather than a more complex approval engine.
|
||||
- Bad, because approval is coarse-grained and may cover more activity than the user expected.
|
||||
- Bad, because it provides less visibility into which provider-owned tools or capabilities will be exercised during the run.
|
||||
|
||||
### Option B: Pre-execution inspection of `call_tool(...)` references before approving `execute_code`
|
||||
|
||||
This option inspects submitted code for statically discoverable `call_tool("tool_name", ...)` references before execution starts and uses that information to shape the approval request.
|
||||
|
||||
- Good, because it can show users more detail up front while still keeping approval at a single pre-execution moment.
|
||||
- Good, because it matches the common case where tool names are spelled out directly in the generated code.
|
||||
- Good, because it can coexist with bundled approval as a more informative variant of the same UX.
|
||||
- Bad, because the analysis is inherently best-effort and cannot reliably predict dynamic behavior.
|
||||
- Bad, because it requires duplicated parsing or inspection logic that does not replace runtime enforcement.
|
||||
|
||||
### Option C: Nested per-tool approvals during `execute_code`
|
||||
|
||||
This option requests approval when sandboxed code actually attempts to invoke a provider-owned tool that requires approval.
|
||||
|
||||
- Good, because it aligns approval with real behavior rather than predicted behavior.
|
||||
- Good, because it gives precise visibility into which provider-owned tools are being used.
|
||||
- Good, because it can allow some tool calls while rejecting others within the same execution.
|
||||
- Bad, because it interrupts long-running CodeAct flows and can degrade the user experience significantly.
|
||||
- Bad, because it requires more complex runtime plumbing and approval UX in both SDKs.
|
||||
- Bad, because repeated approval pauses may make CodeAct less useful for the exact long-running scenarios that motivate this feature.
|
||||
|
||||
## Decision Outcomes
|
||||
|
||||
### Decision 1: Integration seam and public structure
|
||||
|
||||
Chosen option: **Option 1: Standardize on provider-based CodeAct with a shared cross-SDK contract and backend-specific public types**, because it is the only option that maps cleanly to both SDKs, lets us reshape instructions and tools before model invocation, and avoids invasive changes to the existing function invocation loops while still allowing multiple backend-specific providers and multiple runtime modes to fit under the same structure later.
|
||||
|
||||
### Decision 2: Initial approval model
|
||||
|
||||
Chosen option: **Option A: Bundled approval for the `execute_code` invocation**, because it is the smallest approval model that fits both SDKs, works well for long-running CodeAct flows, and does not force us to standardize a more complex inspection or policy engine in the first release.
|
||||
|
||||
This follows the spirit of the current Python tool approval flow, where `FunctionTool` uses `approval_mode="always_require" | "never_require"` and the auto-invocation loop escalates the whole batch when any called tool requires approval.
|
||||
|
||||
### Design summary
|
||||
|
||||
We standardize the **public concept** of CodeAct across SDKs while allowing each SDK to use the extension point that fits it best.
|
||||
|
||||
- Python uses a `ContextProvider`.
|
||||
- .NET uses an `AIContextProvider`.
|
||||
- The term **CodeAct context provider** is used throughout this ADR as a design concept, not as a required public base type. Public SDK APIs should prefer concrete backend-specific types such as `HyperlightCodeActProvider` rather than a public abstract `CodeActContextProvider` or a public `CodeActExecutor` parameter.
|
||||
- CodeAct support should ship as an optional package in each SDK rather than as part of the core package, so users who do not need CodeAct do not take on its installation and dependency footprint. That optional package may still depend on a few small, backward-compatible hooks in the host SDK's core agent pipeline.
|
||||
- There is no separate runtime setup object in the chosen design. Concrete providers manage their provider-owned CodeAct tool registry, file mounts, and outbound network allow-list configuration directly through CRUD-style methods on the provider itself.
|
||||
- At a high level, CodeAct is exposed through backend-specific context providers that contribute an `execute_code` tool, own the CodeAct-specific tool registry, and carry backend capability configuration such as filesystem and network access.
|
||||
- The initial approval model is bundled approval for `execute_code`, using the same `approval_mode="always_require" | "never_require"` vocabulary as regular tools.
|
||||
- The CodeAct provider exposes a default `approval_mode` for `execute_code`. If the provider default is `always_require`, `execute_code` is always treated as `always_require` regardless of the provider-owned tool registry. If the provider default is `never_require`, the effective approval for `execute_code` is derived from the provider-owned CodeAct tool registry captured for the run.
|
||||
- If every provider-owned CodeAct tool in that registry has `approval_mode="never_require"`, `execute_code` is treated as `never_require`. If any provider-owned CodeAct tool in that registry has `approval_mode="always_require"`, `execute_code` is treated as `always_require`, even if the generated code may not end up calling that tool.
|
||||
- Approval is granted before `execute_code` starts, and provider-owned tool calls made from inside that execution run under the same approval.
|
||||
- Direct-only agent tools do not affect the approval of `execute_code`; only the provider-owned CodeAct tool registry participates in that calculation.
|
||||
- This approval model is intentionally conservative. If one sensitive provider-owned tool forces `execute_code` to require approval more often than desired, the mitigation is to keep that tool direct-only or split it into a different provider/tool surface rather than trying to infer per-run tool usage up front.
|
||||
- Configuring filesystem and network capability state on the provider, including adding file mounts or outbound network allow-list entries, is itself the approval for those capabilities in the initial model.
|
||||
- Each `execute_code` invocation must start from a clean execution state; in-memory variables and other ephemeral interpreter/runtime state must not persist across separate calls. When a provider exposes a workspace, mounted files, or a writable artifact/output area, those files are the supported persistence mechanism across calls and are treated as external state rather than interpreter state.
|
||||
- Mutating the provider's tool registry or capability configuration while a run is in flight is allowed, but it only affects subsequent runs. Provider implementations must snapshot the effective state for each run and synchronize concurrent access so shared provider instances remain safe across concurrent runs.
|
||||
- The minimum cross-SDK telemetry contract is that `execute_code` is traced as a normal tool invocation nested inside the surrounding agent run, and provider-owned tool calls made from inside CodeAct continue to emit ordinary tool-invocation telemetry. Backend-specific resource metrics are optional extensions, not a required new top-level cross-SDK event model.
|
||||
- Timeout, out-of-memory, backend crash, and similar sandbox failures are all execution failures of `execute_code` and should surface as structured error results rather than backend-specific public DTOs. Partial textual or file outputs may be returned only when the backend can report them unambiguously; callers must not rely on partial-output recovery as a portable guarantee.
|
||||
- The provider-based structure preserves room for future pre-execution inspection and nested per-tool approvals if later experience shows they are needed.
|
||||
- Concrete backend-specific providers may still use small SDK-local helpers or adapters internally, but that split is an implementation detail rather than a public API requirement.
|
||||
|
||||
Detailed language-specific implementation notes are specified in:
|
||||
|
||||
- [Python implementation](../features/code_act/python-implementation.md)
|
||||
- [.NET implementation](../features/code_act/dotnet-implementation.md)
|
||||
|
||||
### Minimal core hooks required by the optional package
|
||||
|
||||
CodeAct remains optional at the package level, but the optional package depends on a small number of hooks that must live in the host SDK because the agent pipeline owns model invocation and per-run tool resolution.
|
||||
|
||||
- Python depends on the existing `ContextProvider` lifecycle, `SessionContext.extend_instructions(...)`, `SessionContext.extend_tools(...)`, per-run runtime tool access via `SessionContext.options["tools"]`, and the shared `ApprovalMode` vocabulary used by `FunctionTool`.
|
||||
- .NET depends on the existing `AIContextProvider` seam, agent/runtime support for applying providers before model invocation, and the existing chat-client or function-invocation seams that concrete implementations use to contribute `execute_code`.
|
||||
|
||||
These hooks are backward-compatible because they only expose or forward per-run state that core already owns. Behavior changes only when a concrete CodeAct provider opts in and uses them.
|
||||
|
||||
### Concrete provider implementation contract
|
||||
|
||||
The design does not require a public abstract `CodeActContextProvider` base class, but it does require a stable implementation contract for concrete providers.
|
||||
|
||||
- Concrete providers should expose a standard capability surface at construction time, with SDK-appropriate naming for:
|
||||
- approval mode
|
||||
- workspace root
|
||||
- file mounts
|
||||
- allowed outbound targets plus any per-target method or policy restrictions needed by the backend
|
||||
- Separate public `filesystem_mode` / `network_mode` flags are not required by the cross-SDK contract. Filesystem access may be disabled implicitly until a workspace or file mounts are configured, and outbound network may be disabled implicitly until an allow-list or equivalent outbound policy entry is configured.
|
||||
- Concrete providers should expose direct CRUD-style methods for managing the provider-owned CodeAct tool registry, file mounts, and outbound network allow-list configuration, rather than requiring callers to construct a separate runtime setup object.
|
||||
- Concrete providers should implement their host SDK's provider lifecycle hooks to:
|
||||
- build CodeAct instructions,
|
||||
- add `execute_code`,
|
||||
- snapshot the effective CodeAct tool registry and capability settings for the run,
|
||||
- compute the effective approval requirement for `execute_code`,
|
||||
- configure file access and network access for the backend,
|
||||
- prepare or restore execution state,
|
||||
- execute code,
|
||||
- and translate backend output into framework-native content.
|
||||
- Any internal abstract/helper surface shared by multiple concrete providers should standardize responsibilities for:
|
||||
- instruction construction,
|
||||
- file-access configuration,
|
||||
- network-access configuration,
|
||||
- environment preparation/restoration,
|
||||
- code execution,
|
||||
- and output-to-content conversion.
|
||||
- Backend execution output should reuse existing framework-native content/message primitives rather than introducing backend-specific public result DTOs.
|
||||
|
||||
## More Information
|
||||
|
||||
### Related artifacts
|
||||
|
||||
- Python implementation: [`docs/features/code_act/python-implementation.md`](../features/code_act/python-implementation.md)
|
||||
- .NET implementation: [`docs/features/code_act/dotnet-implementation.md`](../features/code_act/dotnet-implementation.md)
|
||||
- Python provider/session APIs: [`python/packages/core/agent_framework/_sessions.py`](../../python/packages/core/agent_framework/_sessions.py)
|
||||
- Python function invocation loop: [`python/packages/core/agent_framework/_tools.py`](../../python/packages/core/agent_framework/_tools.py)
|
||||
- .NET context provider abstraction: [`dotnet/src/Microsoft.Agents.AI.Abstractions/AIContextProvider.cs`](../../dotnet/src/Microsoft.Agents.AI.Abstractions/AIContextProvider.cs)
|
||||
- .NET agent integration for context providers: [`dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgent.cs`](../../dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgent.cs)
|
||||
- Optional .NET chat-client provider decorator: [`dotnet/src/Microsoft.Agents.AI/AIContextProviderDecorators/AIContextProviderChatClient.cs`](../../dotnet/src/Microsoft.Agents.AI/AIContextProviderDecorators/AIContextProviderChatClient.cs)
|
||||
- .NET function invocation middleware seam: [`dotnet/src/Microsoft.Agents.AI/FunctionInvocationDelegatingAgentBuilderExtensions.cs`](../../dotnet/src/Microsoft.Agents.AI/FunctionInvocationDelegatingAgentBuilderExtensions.cs)
|
||||
|
||||
### Related decisions
|
||||
|
||||
- [0015-agent-run-context](0015-agent-run-context.md)
|
||||
- [0016-python-context-middleware](0016-python-context-middleware.md)
|
||||