Compare commits
44 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9e4e4a8fab | |||
| 7b5ef68abc | |||
| a7381d8bef | |||
| d108d4b549 | |||
| fd160a7782 | |||
| ad3c1535c4 | |||
| 10b7d08bff | |||
| fc3111c391 | |||
| 098e521586 | |||
| 7435dd48d0 | |||
| 148f57020a | |||
| 89d19a2370 | |||
| c815902344 | |||
| 074ac68a6c | |||
| d049d94b49 | |||
| 41995e265d | |||
| 2adacb3034 | |||
| 6e3836698b | |||
| 0d3e3504f0 | |||
| 2ba97ebcca | |||
| 2d0555c537 | |||
| 5145d50be8 | |||
| 7f7c88bfa5 | |||
| bcef77af6a | |||
| 54a30571aa | |||
| dc445592ed | |||
| 92823e9e61 | |||
| 7a491f8e76 | |||
| 015e3bcd3b | |||
| 6e95517659 | |||
| 97bb1d588a | |||
| 1fc57c45ee | |||
| b3f8aaa9d7 | |||
| c22fc8d653 | |||
| a3131b8130 | |||
| 3d46595111 | |||
| 205f7bcca8 | |||
| 2048289fb0 | |||
| 699916d639 | |||
| 1ba5cd3f44 | |||
| 1519e50f2f | |||
| bd32e3142c | |||
| 1a698f92ba | |||
| f70c58fa7c |
@@ -1,7 +1,7 @@
|
||||
name: .NET Bug Report
|
||||
description: Report a bug in the Agent Framework .NET SDK
|
||||
title: ".NET: [Bug]: "
|
||||
labels: ["bug", ".NET"]
|
||||
labels: [".NET"]
|
||||
type: bug
|
||||
body:
|
||||
- type: textarea
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
name: Python Bug Report
|
||||
description: Report a bug in the Agent Framework Python SDK
|
||||
title: "Python: [Bug]: "
|
||||
labels: ["bug", "Python"]
|
||||
labels: ["Python"]
|
||||
type: bug
|
||||
body:
|
||||
- type: textarea
|
||||
|
||||
@@ -24,12 +24,14 @@ updates:
|
||||
- ".NET"
|
||||
- "dependencies"
|
||||
|
||||
# Maintain dependencies for python
|
||||
# Maintain dependencies for python.
|
||||
# TODO: Remove these Python Dependabot entries after we have confidence in the
|
||||
# Python dependency-maintenance workflow.
|
||||
- package-ecosystem: "pip"
|
||||
directory: "python/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
day: "thursday"
|
||||
labels:
|
||||
- "python"
|
||||
- "dependencies"
|
||||
@@ -37,7 +39,7 @@ updates:
|
||||
directory: "python/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
day: "thursday"
|
||||
labels:
|
||||
- "python"
|
||||
- "dependencies"
|
||||
|
||||
@@ -90,9 +90,7 @@ jobs:
|
||||
// Check for issue type from issue form dropdown
|
||||
const issueTypeField = getFormFieldValue(body, 'Type of Issue')
|
||||
if (issueTypeField) {
|
||||
if (issueTypeField === 'Bug') {
|
||||
labels.push("bug")
|
||||
} else if (issueTypeField === 'Feature Request') {
|
||||
if (issueTypeField === 'Feature Request') {
|
||||
labels.push("enhancement")
|
||||
} else if (issueTypeField === 'Question') {
|
||||
labels.push("question")
|
||||
|
||||
@@ -113,8 +113,8 @@ jobs:
|
||||
- name: Run markdown code lint
|
||||
run: uv run poe markdown-code-lint
|
||||
|
||||
mypy:
|
||||
name: Mypy Checks
|
||||
test-typing:
|
||||
name: Test Typing Checks
|
||||
if: "!cancelled()"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -139,7 +139,5 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
- name: Run Mypy
|
||||
env:
|
||||
GITHUB_BASE_REF: ${{ github.event.pull_request.base.ref || github.base_ref || 'main' }}
|
||||
run: uv run python scripts/workspace_poe_tasks.py ci-mypy
|
||||
- name: Run tests/samples type checkers (mypy, pyrefly, ty)
|
||||
run: uv run python scripts/workspace_poe_tasks.py ci-test-typing
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
name: Python - Dependency Maintenance
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 4 * * 1"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: python-dependency-maintenance
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
dependency-maintenance:
|
||||
name: Dependency Maintenance
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Match the existing Python dependency maintenance workflows. Reevaluate if package
|
||||
# installability starts differing across supported Python versions.
|
||||
UV_PYTHON: "3.13"
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Set dependency release cutoff
|
||||
run: |
|
||||
cutoff="$(date -u -d '7 days ago' '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
echo "DEPENDENCY_RELEASE_CUTOFF=${cutoff}" >> "$GITHUB_ENV"
|
||||
echo "Using dependency release cutoff: ${cutoff}"
|
||||
|
||||
- name: Repin dev dependency declarations
|
||||
run: uv run poe upgrade-dev-dependency-pins
|
||||
working-directory: ./python
|
||||
|
||||
- name: Refresh lockfile after dev pin updates
|
||||
run: uv lock
|
||||
working-directory: ./python
|
||||
|
||||
- name: Save dev dependency changes
|
||||
run: |
|
||||
DEV_PATCH="${RUNNER_TEMP}/python-dev-dependency-updates.patch"
|
||||
git diff -- python/pyproject.toml "python/packages/*/pyproject.toml" python/uv.lock > "${DEV_PATCH}"
|
||||
if [ -s "${DEV_PATCH}" ]; then
|
||||
echo "has_dev_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_dev_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "patch=${DEV_PATCH}" >> "$GITHUB_OUTPUT"
|
||||
id: dev_changes
|
||||
|
||||
- name: Run dependency bounds test scenarios
|
||||
id: validate_bounds_test
|
||||
continue-on-error: true
|
||||
run: uv run poe validate-dependency-bounds-test --package "*"
|
||||
working-directory: ./python
|
||||
|
||||
- name: Run dependency upper-bound validation
|
||||
id: validate_ranges
|
||||
if: steps.validate_bounds_test.outcome == 'success'
|
||||
continue-on-error: true
|
||||
run: uv run poe validate-dependency-bounds-project --mode upper --package "*"
|
||||
working-directory: ./python
|
||||
|
||||
- name: Upload dependency validation reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: dependency-maintenance-results
|
||||
path: |
|
||||
python/scripts/dependencies/dependency-bounds-test-results.json
|
||||
python/scripts/dependencies/dependency-range-results.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Create issue for failed dependency bounds test
|
||||
if: steps.validate_bounds_test.outcome != 'success'
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const fs = require("fs")
|
||||
const reportPath = "python/scripts/dependencies/dependency-bounds-test-results.json"
|
||||
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const openIssueTitles = new Set(
|
||||
openIssues.filter((issue) => !issue.pull_request).map((issue) => issue.title)
|
||||
)
|
||||
|
||||
const formatError = (message) => String(message || "No error output captured.").replace(/```/g, "'''")
|
||||
const title = "Dependency bounds test failed"
|
||||
if (openIssueTitles.has(title)) {
|
||||
core.info(`Issue already exists: ${title}`)
|
||||
return
|
||||
}
|
||||
|
||||
const bodyLines = [
|
||||
"Automated dependency bounds test mode failed before dependency upper-bound validation could run.",
|
||||
"",
|
||||
"The weekly dependency maintenance workflow kept only dev dependency updates for the generated PR, if any, and skipped dependency range updates for this run.",
|
||||
"",
|
||||
]
|
||||
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
|
||||
const failedScenarios = (report.scenarios ?? []).filter((scenario) => scenario.status === "failed")
|
||||
for (const scenario of failedScenarios) {
|
||||
bodyLines.push(`### ${scenario.name} scenario (${scenario.resolution})`)
|
||||
const failedPackages = (scenario.packages ?? []).filter((pkg) => pkg.status === "failed")
|
||||
for (const pkg of failedPackages.slice(0, 10)) {
|
||||
bodyLines.push(
|
||||
"",
|
||||
`- Package: \`${pkg.package_name}\``,
|
||||
`- Project path: \`${pkg.project_path}\``,
|
||||
"",
|
||||
"```",
|
||||
formatError(pkg.error).slice(0, 3500),
|
||||
"```"
|
||||
)
|
||||
}
|
||||
if (failedPackages.length > 10) {
|
||||
bodyLines.push("", `_Additional failed packages omitted: ${failedPackages.length - 10}_`)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
bodyLines.push(`No dependency bounds test report was found at \`${reportPath}\`.`)
|
||||
}
|
||||
|
||||
bodyLines.push("", `Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`)
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title,
|
||||
body: bodyLines.join("\n"),
|
||||
})
|
||||
core.info(`Created issue: ${title}`)
|
||||
|
||||
- name: Create issues for failed dependency candidates
|
||||
if: always()
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const fs = require("fs")
|
||||
const reportPath = "python/scripts/dependencies/dependency-range-results.json"
|
||||
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
core.info(`No dependency range report found at ${reportPath}`)
|
||||
return
|
||||
}
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
|
||||
const dependencyFailures = []
|
||||
|
||||
for (const packageResult of report.packages ?? []) {
|
||||
for (const dependency of packageResult.dependencies ?? []) {
|
||||
const candidateVersions = new Set(dependency.candidate_versions ?? [])
|
||||
const failedAttempts = (dependency.attempts ?? []).filter(
|
||||
(attempt) => attempt.status === "failed" && candidateVersions.has(attempt.trial_upper)
|
||||
)
|
||||
if (!failedAttempts.length) {
|
||||
continue
|
||||
}
|
||||
|
||||
const failuresByVersion = new Map()
|
||||
for (const attempt of failedAttempts) {
|
||||
const version = attempt.trial_upper || "unknown"
|
||||
if (!failuresByVersion.has(version)) {
|
||||
failuresByVersion.set(version, attempt.error || "No error output captured.")
|
||||
}
|
||||
}
|
||||
|
||||
dependencyFailures.push({
|
||||
packageName: packageResult.package_name,
|
||||
projectPath: packageResult.project_path,
|
||||
dependencyName: dependency.name,
|
||||
originalRequirements: dependency.original_requirements ?? [],
|
||||
finalRequirements: dependency.final_requirements ?? [],
|
||||
failedVersions: [...failuresByVersion.entries()].map(([version, error]) => ({ version, error })),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!dependencyFailures.length) {
|
||||
core.info("No failing dependency candidates found.")
|
||||
return
|
||||
}
|
||||
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const openIssueTitles = new Set(
|
||||
openIssues.filter((issue) => !issue.pull_request).map((issue) => issue.title)
|
||||
)
|
||||
|
||||
const formatError = (message) => String(message || "No error output captured.").replace(/```/g, "'''")
|
||||
|
||||
for (const failure of dependencyFailures) {
|
||||
const title = `Dependency validation failed: ${failure.dependencyName} (${failure.packageName})`
|
||||
if (openIssueTitles.has(title)) {
|
||||
core.info(`Issue already exists: ${title}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const visibleFailures = failure.failedVersions.slice(0, 5)
|
||||
const omittedCount = failure.failedVersions.length - visibleFailures.length
|
||||
const failureDetails = visibleFailures
|
||||
.map(
|
||||
(entry) =>
|
||||
`- \`${entry.version}\`\n\n\`\`\`\n${formatError(entry.error).slice(0, 3500)}\n\`\`\``
|
||||
)
|
||||
.join("\n\n")
|
||||
|
||||
const body = [
|
||||
"Automated dependency range validation found candidate versions that failed checks.",
|
||||
"",
|
||||
`- Package: \`${failure.packageName}\``,
|
||||
`- Project path: \`${failure.projectPath}\``,
|
||||
`- Dependency: \`${failure.dependencyName}\``,
|
||||
`- Original requirements: ${
|
||||
failure.originalRequirements.length
|
||||
? failure.originalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
`- Final requirements after run: ${
|
||||
failure.finalRequirements.length
|
||||
? failure.finalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
"",
|
||||
"### Failed versions and errors",
|
||||
failureDetails,
|
||||
omittedCount > 0 ? `\n_Additional failed versions omitted: ${omittedCount}_` : "",
|
||||
"",
|
||||
`Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
|
||||
].join("\n")
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title,
|
||||
body,
|
||||
})
|
||||
openIssueTitles.add(title)
|
||||
core.info(`Created issue: ${title}`)
|
||||
}
|
||||
|
||||
- name: Keep only dev updates when range validation fails
|
||||
if: steps.validate_bounds_test.outcome != 'success' || steps.validate_ranges.outcome != 'success'
|
||||
env:
|
||||
DEV_PATCH: ${{ steps.dev_changes.outputs.patch }}
|
||||
HAS_DEV_CHANGES: ${{ steps.dev_changes.outputs.has_dev_changes }}
|
||||
run: |
|
||||
git restore python/pyproject.toml python/packages/*/pyproject.toml python/uv.lock
|
||||
if [ "${HAS_DEV_CHANGES}" = "true" ]; then
|
||||
git apply "${DEV_PATCH}"
|
||||
fi
|
||||
|
||||
- name: Refresh lockfile after dependency range updates
|
||||
if: steps.validate_bounds_test.outcome == 'success' && steps.validate_ranges.outcome == 'success'
|
||||
run: uv lock
|
||||
working-directory: ./python
|
||||
|
||||
- name: Install final dependency set
|
||||
run: uv run poe install
|
||||
working-directory: ./python
|
||||
|
||||
- name: Run final checks
|
||||
run: uv run poe check
|
||||
working-directory: ./python
|
||||
|
||||
- name: Run final typing
|
||||
run: uv run poe typing
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dependency updates
|
||||
id: commit_updates
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-maintenance"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/pyproject.toml python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "Python: chore: update dependencies"
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update pull request with GitHub CLI
|
||||
if: steps.commit_updates.outputs.has_changes == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-maintenance"
|
||||
PR_TITLE="Python: chore: update dependencies"
|
||||
PR_BODY_FILE="$(mktemp)"
|
||||
|
||||
cat > "${PR_BODY_FILE}" <<'EOF'
|
||||
### Motivation & Context
|
||||
|
||||
This automated update keeps Python dependency metadata coherent across the uv workspace. Python dependencies can be declared in multiple `pyproject.toml` files, but the workspace has one shared `python/uv.lock`, so dependency maintenance should update and validate them together instead of through per-manifest Dependabot PRs.
|
||||
|
||||
### Description & Review Guide
|
||||
|
||||
- **What are the major changes?** Refresh Python dev dependency pins, update package dependency ranges when the bounds tooling succeeds, and refresh `python/uv.lock`.
|
||||
- **What is the impact of these changes?** Keeps the Python workspace dependency set current while producing at most one dependency PR for the week. If dependency range validation fails, this PR contains only the dev dependency updates that still pass final validation, and separate issues track failed range candidates.
|
||||
- **What do you want reviewers to focus on?** Review the generated dependency metadata changes and any dependency-range updates for package-specific compatibility concerns.
|
||||
<!-- NOTE FOR AUTOMATED/AI REVIEWERS: the "What do you want reviewers to focus on?"
|
||||
item above is intended for human reviewers only. Automated/AI reviewers should
|
||||
ignore it and review the entire change rather than narrowing scope to it. -->
|
||||
|
||||
|
||||
### Related Issue
|
||||
|
||||
No linked issue; this PR is generated by scheduled Python dependency maintenance.
|
||||
|
||||
### Contribution Checklist
|
||||
|
||||
- [x] The code builds clean without any errors or warnings
|
||||
- [x] All unit tests pass, and I have added new tests where possible
|
||||
- [x] The PR follows the [Contribution Guidelines](https://github.com/microsoft/agent-framework/blob/main/CONTRIBUTING.md)
|
||||
- [ ] This PR is linked to an issue and there is no other open PR for this issue (see Related Issue above).
|
||||
- [x] **This is not a breaking change.** If it _is_ a breaking change, add the `breaking change` label (or add "[BREAKING]" to the title prefix, before or after any language prefix) — a workflow keeps the label and title prefix in sync automatically.
|
||||
EOF
|
||||
|
||||
PR_NUMBER="$(gh pr list --head "${BRANCH}" --base main --state open --json number --jq '.[0].number')"
|
||||
if [ -n "${PR_NUMBER}" ]; then
|
||||
gh pr edit "${PR_NUMBER}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
else
|
||||
gh pr create --base main --head "${BRANCH}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
fi
|
||||
@@ -1,216 +0,0 @@
|
||||
# Probe the highest allowed dependency versions, then open issues/PRs from the passing updates.
|
||||
name: Python - Dependency Range Validation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
dependency-range-validation:
|
||||
name: Dependency Range Validation
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# For now only run 3.13, if we do encounter situations where there are mismatches between packages and python versions (other then 3.10 and 3.14 which are known to not be able to install everything)
|
||||
# then we will have to reevaluate.
|
||||
UV_PYTHON: "3.13"
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Run dependency range validation
|
||||
id: validate_ranges
|
||||
# Keep workflow running so we can still publish diagnostics from this run.
|
||||
continue-on-error: true
|
||||
run: uv run poe validate-dependency-bounds-project --mode upper --package "*"
|
||||
working-directory: ./python
|
||||
|
||||
- name: Upload dependency range report
|
||||
# Always publish the report so failures are inspectable even when validation fails.
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: dependency-range-results
|
||||
path: python/scripts/dependencies/dependency-range-results.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Create issues for failed dependency candidates
|
||||
# Always process the report so failed candidates create actionable tracking issues.
|
||||
if: always()
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
script: |
|
||||
const fs = require("fs")
|
||||
const reportPath = "python/scripts/dependencies/dependency-range-results.json"
|
||||
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
core.warning(`No dependency range report found at ${reportPath}`)
|
||||
return
|
||||
}
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
|
||||
const dependencyFailures = []
|
||||
|
||||
for (const packageResult of report.packages ?? []) {
|
||||
for (const dependency of packageResult.dependencies ?? []) {
|
||||
const candidateVersions = new Set(dependency.candidate_versions ?? [])
|
||||
const failedAttempts = (dependency.attempts ?? []).filter(
|
||||
(attempt) => attempt.status === "failed" && candidateVersions.has(attempt.trial_upper)
|
||||
)
|
||||
if (!failedAttempts.length) {
|
||||
continue
|
||||
}
|
||||
|
||||
const failuresByVersion = new Map()
|
||||
for (const attempt of failedAttempts) {
|
||||
const version = attempt.trial_upper || "unknown"
|
||||
if (!failuresByVersion.has(version)) {
|
||||
failuresByVersion.set(version, attempt.error || "No error output captured.")
|
||||
}
|
||||
}
|
||||
|
||||
dependencyFailures.push({
|
||||
packageName: packageResult.package_name,
|
||||
projectPath: packageResult.project_path,
|
||||
dependencyName: dependency.name,
|
||||
originalRequirements: dependency.original_requirements ?? [],
|
||||
finalRequirements: dependency.final_requirements ?? [],
|
||||
failedVersions: [...failuresByVersion.entries()].map(([version, error]) => ({ version, error })),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!dependencyFailures.length) {
|
||||
core.info("No failing dependency candidates found.")
|
||||
return
|
||||
}
|
||||
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const openIssueTitles = new Set(
|
||||
openIssues.filter((issue) => !issue.pull_request).map((issue) => issue.title)
|
||||
)
|
||||
|
||||
const formatError = (message) => String(message || "No error output captured.").replace(/```/g, "'''")
|
||||
|
||||
for (const failure of dependencyFailures) {
|
||||
const title = `Dependency validation failed: ${failure.dependencyName} (${failure.packageName})`
|
||||
if (openIssueTitles.has(title)) {
|
||||
core.info(`Issue already exists: ${title}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const visibleFailures = failure.failedVersions.slice(0, 5)
|
||||
const omittedCount = failure.failedVersions.length - visibleFailures.length
|
||||
const failureDetails = visibleFailures
|
||||
.map(
|
||||
(entry) =>
|
||||
`- \`${entry.version}\`\n\n\`\`\`\n${formatError(entry.error).slice(0, 3500)}\n\`\`\``
|
||||
)
|
||||
.join("\n\n")
|
||||
|
||||
const body = [
|
||||
"Automated dependency range validation found candidate versions that failed checks.",
|
||||
"",
|
||||
`- Package: \`${failure.packageName}\``,
|
||||
`- Project path: \`${failure.projectPath}\``,
|
||||
`- Dependency: \`${failure.dependencyName}\``,
|
||||
`- Original requirements: ${
|
||||
failure.originalRequirements.length
|
||||
? failure.originalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
`- Final requirements after run: ${
|
||||
failure.finalRequirements.length
|
||||
? failure.finalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
"",
|
||||
"### Failed versions and errors",
|
||||
failureDetails,
|
||||
omittedCount > 0 ? `\n_Additional failed versions omitted: ${omittedCount}_` : "",
|
||||
"",
|
||||
`Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
|
||||
].join("\n")
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title,
|
||||
body,
|
||||
})
|
||||
openIssueTitles.add(title)
|
||||
core.info(`Created issue: ${title}`)
|
||||
}
|
||||
|
||||
- name: Refresh lockfile
|
||||
# Only refresh lockfile after a clean validation to avoid committing known-bad ranges.
|
||||
if: steps.validate_ranges.outcome == 'success'
|
||||
run: uv lock --upgrade
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dependency updates
|
||||
id: commit_updates
|
||||
if: steps.validate_ranges.outcome == 'success'
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-range-updates"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "chore: update dependency ranges"
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update pull request with GitHub CLI
|
||||
# Only open/update PRs for validated updates to keep automation branches trustworthy.
|
||||
if: steps.validate_ranges.outcome == 'success' && steps.commit_updates.outputs.has_changes == 'true'
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-range-updates"
|
||||
PR_TITLE="Python: chore: update dependency ranges"
|
||||
PR_BODY_FILE="$(mktemp)"
|
||||
|
||||
cat > "${PR_BODY_FILE}" <<'EOF'
|
||||
This PR was generated by the dependency range validation workflow.
|
||||
|
||||
- Ran `uv run poe validate-dependency-bounds-project --mode upper --package "*"`
|
||||
- Updated package dependency bounds
|
||||
- Refreshed `python/uv.lock` with `uv lock --upgrade`
|
||||
EOF
|
||||
|
||||
PR_NUMBER="$(gh pr list --head "${BRANCH}" --base main --state open --json number --jq '.[0].number')"
|
||||
if [ -n "${PR_NUMBER}" ]; then
|
||||
gh pr edit "${PR_NUMBER}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
else
|
||||
gh pr create --base main --head "${BRANCH}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
fi
|
||||
@@ -1,91 +0,0 @@
|
||||
name: Python - Dev Dependency Upgrade
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
upgrade-dev-dependencies:
|
||||
name: Upgrade Dev Dependencies
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
UV_PYTHON: "3.13"
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Upgrade dev dependencies and validate workspace
|
||||
run: uv run poe upgrade-dev-dependencies
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dev dependency updates
|
||||
id: commit_updates
|
||||
run: |
|
||||
BRANCH="automation/python-dev-dependency-updates"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/pyproject.toml python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dev dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -F- <<'EOF'
|
||||
Python: chore: upgrade dev dependencies
|
||||
EOF
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update pull request with GitHub CLI
|
||||
if: steps.commit_updates.outputs.has_changes == 'true'
|
||||
run: |
|
||||
BRANCH="automation/python-dev-dependency-updates"
|
||||
PR_TITLE="Python: chore: upgrade dev dependencies"
|
||||
PR_BODY_FILE="$(mktemp)"
|
||||
|
||||
cat > "${PR_BODY_FILE}" <<'EOF'
|
||||
### Motivation and Context
|
||||
|
||||
This automated update refreshes Python dev dependency pins across the workspace and reruns the repo validation gates before opening a pull request.
|
||||
|
||||
### Description
|
||||
|
||||
- Ran `uv run poe upgrade-dev-dependencies`
|
||||
- Refreshed dev dependency pins in workspace `pyproject.toml` files
|
||||
- Refreshed `python/uv.lock` with `uv lock --upgrade`
|
||||
- Reinstalled from the frozen lockfile and reran `check`, `typing`, and `test`
|
||||
|
||||
### Contribution Checklist
|
||||
|
||||
- [x] The code builds clean without any errors or warnings
|
||||
- [x] The PR follows the [Contribution Guidelines](https://github.com/microsoft/agent-framework/blob/main/CONTRIBUTING.md)
|
||||
- [x] All unit tests pass, and I have added new tests where possible
|
||||
- [ ] **Is this a breaking change?** If yes, add "[BREAKING]" prefix to the title of the PR.
|
||||
EOF
|
||||
|
||||
PR_NUMBER="$(gh pr list --head "${BRANCH}" --base main --state open --json number --jq '.[0].number')"
|
||||
if [ -n "${PR_NUMBER}" ]; then
|
||||
gh pr edit "${PR_NUMBER}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
else
|
||||
gh pr create --base main --head "${BRANCH}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
fi
|
||||
@@ -92,9 +92,6 @@ jobs:
|
||||
- name: Run lab type checking
|
||||
run: cd packages/lab && uv run poe pyright
|
||||
|
||||
- name: Run lab mypy
|
||||
run: cd packages/lab && uv run poe mypy
|
||||
|
||||
# Surface failing tests
|
||||
- name: Surface failing tests
|
||||
if: always()
|
||||
|
||||
@@ -30,21 +30,31 @@ jobs:
|
||||
merge-multiple: true
|
||||
- name: Display structure of downloaded files
|
||||
run: ls
|
||||
- name: Read and set PR number
|
||||
# Need to read the PR number from the file saved in the previous workflow
|
||||
# because the workflow_run event does not have access to the PR number
|
||||
# The PR number is needed to post the comment on the PR
|
||||
- name: Read and validate PR number
|
||||
# Keep the artifact handoff aligned with the workflow run that produced it.
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
run: |
|
||||
if [ ! -s pr_number ]; then
|
||||
echo "PR number file 'pr_number' is missing or empty"
|
||||
exit 1
|
||||
fi
|
||||
PR_NUMBER=$(cat pr_number)
|
||||
if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then
|
||||
|
||||
ARTIFACT_PR_NUMBER=$(cat pr_number)
|
||||
if ! [[ "$ARTIFACT_PR_NUMBER" =~ ^[0-9]+$ ]]; then
|
||||
echo "::error::PR number file contains invalid content"
|
||||
exit 1
|
||||
fi
|
||||
echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV"
|
||||
|
||||
PR_HEAD_SHA=$(gh pr view "$ARTIFACT_PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid')
|
||||
if [ "$PR_HEAD_SHA" != "$RUN_HEAD_SHA" ]; then
|
||||
echo "::error::PR head SHA does not match the triggering workflow run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PR_NUMBER=$ARTIFACT_PR_NUMBER" >> "$GITHUB_ENV"
|
||||
- name: Pytest coverage comment
|
||||
id: coverageComment
|
||||
uses: MishaKav/pytest-coverage-comment@26f986d2599c288bb62f623d29c2da98609e9cd4 # v1.6.0
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
---
|
||||
status: accepted
|
||||
contact: eavanvalkenburg
|
||||
date: 2026-06-11
|
||||
deciders: eavanvalkenburg
|
||||
---
|
||||
|
||||
# Python minimal hosting core and pluggable channels
|
||||
|
||||
## Context and Problem Statement
|
||||
|
||||
Agent Framework has several protocol-specific hosting surfaces. App authors who want one agent or workflow on multiple protocols must compose servers, routes, middleware, session handling, and lifecycle code by hand.
|
||||
|
||||
We will introduce a small Python hosting core that owns the common server shape and leaves protocol details inside channel packages. The first public contract must be intentionally narrow so Python can ship a base contract before adding identity linking, proactive delivery, or multicast behavior. Other language implementations may reuse the same conceptual boundary, but this ADR records the Python decision.
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- Keep the first host easy to explain: one app, one hostable target, one or more channels.
|
||||
- Reuse Agent Framework's existing agent, workflow, session, history, and checkpoint primitives.
|
||||
- Let channel packages own protocol parsing, protocol responses, authentication details, and native command surfaces.
|
||||
- Make session continuity explicit through a channel-supplied `ChannelSession(isolation_key=...)`.
|
||||
- Avoid approving cross-channel identity and delivery semantics before their safety model is reviewed.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. Keep only protocol-specific hosts.
|
||||
2. Ship a large hosting core with identity linking, authorization, background delivery, active-channel routing, and multicast in v1.
|
||||
3. Ship a minimal host/channel core now and track linking/multicast as follow-up work.
|
||||
|
||||
### Keep only protocol-specific hosts
|
||||
|
||||
- Good: no new abstraction or package surface.
|
||||
- Neutral: each protocol can continue evolving independently.
|
||||
- Bad: every multi-channel app still has to compose servers, lifecycle, and session handling by hand.
|
||||
|
||||
### Ship the large cross-channel host in v1
|
||||
|
||||
- Good: the richest cross-channel scenarios are available immediately.
|
||||
- Neutral: the host becomes the natural place to demonstrate identity and delivery policy.
|
||||
- Bad: v1 becomes a security-sensitive identity and delivery system before the safety model is reviewed.
|
||||
|
||||
### Ship the minimal core now
|
||||
|
||||
- Good: the host/channel boundary can be implemented, tested, and explained without solving linking and durable delivery at the same time.
|
||||
- Neutral: apps that need richer behavior must build it locally or wait for ADR-0028 follow-up work.
|
||||
- Bad: proactive delivery and multicast scenarios are deliberately absent from v1.
|
||||
|
||||
## Decision Outcome
|
||||
|
||||
Chosen option: **minimal host/channel core now, follow-up enhancements later**.
|
||||
|
||||
`AgentFrameworkHost` owns:
|
||||
|
||||
- one application object,
|
||||
- one hostable target (`SupportsAgentRun` agent-compatible object or a `Workflow`), and
|
||||
- one or more channels.
|
||||
|
||||
Channels own:
|
||||
|
||||
- contributed routes, middleware, commands, and lifecycle callbacks,
|
||||
- protocol-native request parsing into `ChannelRequest`,
|
||||
- protocol-native rendering of the originating response, and
|
||||
- any channel-specific authentication or signature validation.
|
||||
|
||||
The host owns:
|
||||
|
||||
- route/lifecycle aggregation,
|
||||
- invocation of the target,
|
||||
- `ChannelSession(isolation_key=...)` to `AgentSession` resolution and caching,
|
||||
- `reset_session(isolation_key=...)`,
|
||||
- host-level middleware, including Foundry isolation middleware only when the Foundry hosting environment flag is present,
|
||||
- invocation of per-channel hooks (`ChannelRunHook`, `ChannelResponseHook`, `ChannelStreamUpdateHook`), and
|
||||
- workflow checkpoint wiring through an explicit `checkpoint_location`.
|
||||
|
||||
`ChannelIdentity`, when present, is request metadata only. In v1 it is not a linking, authorization, or delivery key.
|
||||
|
||||
### Trust boundary for `isolation_key`
|
||||
|
||||
The host treats `ChannelSession.isolation_key` as a session partition key, not as proof of identity. Channels or host middleware must authenticate and authorize any externally supplied value before passing it to the host. For example, a Responses caller must not be allowed to choose an arbitrary `previous_response_id` or header-derived key unless the platform or middleware has already established that the caller owns that conversation. The host deliberately does not infer that trust from the string itself.
|
||||
|
||||
### Hook ownership
|
||||
|
||||
Channels provide hook configuration and protocol-native context. The host invokes those hooks as part of the common invocation pipeline:
|
||||
|
||||
- `ChannelRunHook` runs after channel parsing and before target invocation.
|
||||
- `ChannelResponseHook` runs after target invocation and before the originating channel serializes its response.
|
||||
- `ChannelStreamUpdateHook` is applied by the host while the channel consumes streamed updates because streaming serialization is protocol-specific.
|
||||
|
||||
`ChannelStreamUpdateHook` is an update hook, not a final-response sanitizer. Channels that use it for redaction or filtering must also apply equivalent policy to any final response they render. Channels choose whether the response is streaming before run hooks execute.
|
||||
|
||||
This keeps hook call conventions centralized while leaving protocol payload parsing and response formatting in channel packages.
|
||||
|
||||
### State owned by v1
|
||||
|
||||
`state_dir` is limited to host-owned local files for reset-session aliases and workflow checkpoint path derivation. It does not store linked identities, active-channel state, response-routing state, continuation records, durable runner queues, or delivery attempts. Those storage concerns belong to ADR-0028.
|
||||
|
||||
## Non-goals for v1
|
||||
|
||||
The following are deliberately **not** part of the v1 contract:
|
||||
|
||||
- cross-channel identity linking (`IdentityLinker`, `local_identity_link`, or `agent-framework-hosting-entra`),
|
||||
- identity allowlists or authorization policy (`IdentityAllowlist`, `AuthPolicy`),
|
||||
- response routing beyond the originating channel (`ResponseTarget`, active channel, specific linked channel, `all_linked`),
|
||||
- push or payload codecs (`ChannelPush`, `ChannelPushCodec`),
|
||||
- background/continuation delivery,
|
||||
- durable task runners (`DurableTaskRunner`, `InProcessTaskRunner`),
|
||||
- retry/replay policy (`RetryPolicy`),
|
||||
- fan-out, multicast, or all-linked delivery,
|
||||
- confidentiality tiers and `LinkPolicy`, and
|
||||
- a host-level multi-agent router.
|
||||
|
||||
These areas are follow-up enhancements covered by [ADR-0028](0028-hosting-linking-multicast-enhancements.md). They are not prerequisites for shipping or using the v1 host.
|
||||
|
||||
## Consequences
|
||||
|
||||
Positive:
|
||||
|
||||
- The host/channel model can be implemented and tested without designing a security-sensitive identity graph.
|
||||
- Existing and new channel packages can share one Starlette app, middleware stack, lifecycle, and target invocation path.
|
||||
- Session continuity is explicit and debuggable: two channels share history only when they produce the same `isolation_key`.
|
||||
- Hook invocation is centralized in the host, so channels do not each invent the call convention.
|
||||
|
||||
Negative:
|
||||
|
||||
- Apps that need OAuth linking, allowlists, proactive messages, or multicast must continue to implement those behaviors outside the v1 host.
|
||||
- Some richer cross-channel scenarios from the original design move to a separate decision and validation cycle.
|
||||
- The host must document `isolation_key` trust clearly because it now provides the shared session boundary.
|
||||
|
||||
## Validation Gates
|
||||
|
||||
Before this ADR is accepted:
|
||||
|
||||
- A sample can expose one target on multiple channels with one `AgentFrameworkHost` and no handwritten Starlette route composition.
|
||||
- Built-in channel tests prove that routes, commands, startup, and shutdown callbacks are contributed by channels and aggregated by the host.
|
||||
- Session tests prove that identical `ChannelSession.isolation_key` values resolve to the same cached `AgentSession`, and `reset_session` rotates that mapping.
|
||||
- Channel tests prove that each channel renders only its own originating response; there is no host-level push, multicast, or active-channel delivery path.
|
||||
- Workflow tests or samples use an explicit `checkpoint_location`.
|
||||
- Foundry isolation middleware is documented and covered by integration or contract tests, including the non-Foundry case where raw isolation headers are ignored.
|
||||
- The v1 API and packages do not expose the removed symbols or packages listed in [Non-goals for v1](#non-goals-for-v1).
|
||||
- The Python spec is updated to match this simplified contract and uses "public", "stable", or "released" terminology for Agent Framework APIs.
|
||||
|
||||
## More Information
|
||||
|
||||
- Follow-up linking and multicast ADR: [ADR-0028](0028-hosting-linking-multicast-enhancements.md)
|
||||
@@ -0,0 +1,132 @@
|
||||
---
|
||||
status: proposed
|
||||
contact: eavanvalkenburg
|
||||
date: 2026-06-11
|
||||
deciders: eavanvalkenburg
|
||||
---
|
||||
|
||||
# Hosting linking and multicast enhancements
|
||||
|
||||
## Context and Problem Statement
|
||||
|
||||
[ADR-0027](0027-hosting-channels.md) defines the minimal v1 hosting core: originating-channel responses, explicit `ChannelSession.isolation_key`, and no host-level identity linking, push, multicast, background delivery, or durable runners.
|
||||
|
||||
This ADR tracks the richer cross-channel behaviors that were removed from v1. These enhancements are **follow-up work** and are **not prerequisites** for shipping, using, or stabilizing the v1 host/channel core.
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- Cross-channel continuity must not create accidental cross-user, cross-tenant, or cross-channel data leaks.
|
||||
- Non-originating delivery must be observable, idempotent, retryable, and supportable.
|
||||
- Protocol payloads must remain channel-native while still being safe to persist and replay.
|
||||
- App authors need opt-in policy controls, not hidden defaults.
|
||||
- The enhancement stack should layer on top of the v1 host without reshaping the minimal channel contract.
|
||||
|
||||
## Enhancement Areas
|
||||
|
||||
The follow-up design should cover these capabilities together because they share identity, storage, delivery, and replay concerns:
|
||||
|
||||
- **Cross-channel identity linking** — a user can connect multiple `ChannelIdentity` values to one channel-neutral `isolation_key`.
|
||||
- **Authorization and allowlist policy** — channels or hosts can require verified identity, allow specific native identities or claims, and deny unknown callers.
|
||||
- **Non-originating response delivery** — a run can respond somewhere other than the request's originating protocol when explicitly configured.
|
||||
- **Active-channel routing** — delivery can target the most recently observed linked channel for an `isolation_key`.
|
||||
- **Multicast / all-linked delivery** — delivery can fan out to every linked channel or a selected set.
|
||||
- **Background runs and continuation tokens** — long-running requests can return immediately and complete later, with a polling/status fallback.
|
||||
- **Durable delivery runners** — delivery work can survive process restarts and support dead-letter handling.
|
||||
- **Retry and replay semantics** — delivery attempts are bounded, deduplicated, and safe to replay.
|
||||
- **Payload serialization** — channel-specific payloads can be persisted, redacted, versioned, and reconstructed without losing protocol fidelity.
|
||||
|
||||
Candidate API names from the broader design (`IdentityLinker`, `IdentityAllowlist`, `AuthPolicy`, `ResponseTarget`, `ChannelPush`, `ChannelPushCodec`, `DurableTaskRunner`, `InProcessTaskRunner`, `RetryPolicy`, `LinkPolicy`) remain design vocabulary for this ADR. They are not approved v1 APIs.
|
||||
|
||||
## Considered Options
|
||||
|
||||
### Option A — Leave all behavior to applications
|
||||
|
||||
Applications implement linking, authorization, push, retry, and serialization independently.
|
||||
|
||||
- Good: the hosting core stays very small.
|
||||
- Neutral: advanced apps can still build what they need.
|
||||
- Bad: every app must solve the same security and delivery problems, likely inconsistently.
|
||||
|
||||
### Option B — Add the full enhancement stack to v1
|
||||
|
||||
The first host release includes linking, authorization, active channel, multicast, background runs, durable runners, and codecs.
|
||||
|
||||
- Good: the original cross-channel experience is available immediately.
|
||||
- Neutral: samples can demonstrate rich end-to-end flows.
|
||||
- Bad: v1 becomes security-sensitive, storage-heavy, and harder to stabilize.
|
||||
|
||||
### Option C — Layer opt-in enhancement packages after v1
|
||||
|
||||
Ship the minimal host first, then add linking, authorization, and delivery packages behind explicit configuration.
|
||||
|
||||
- Good: v1 remains simple while leaving room for a reviewed, supportable enhancement stack.
|
||||
- Neutral: apps that need advanced delivery wait for follow-up packages.
|
||||
- Bad: the first release does not satisfy proactive or all-linked scenarios.
|
||||
|
||||
### Option D — Build only platform-specific integrations
|
||||
|
||||
Implement linking and proactive delivery separately in Telegram, Activity Protocol, Discord, and future channels.
|
||||
|
||||
- Good: each package can match its protocol exactly.
|
||||
- Neutral: some shared abstractions may emerge later.
|
||||
- Bad: cross-channel behavior becomes fragmented and hard to reason about.
|
||||
|
||||
## Decision Outcome
|
||||
|
||||
Proposed direction: **Option C — layered opt-in enhancement packages after v1**.
|
||||
|
||||
The minimal host remains the foundation. Follow-up packages may add linking, authorization, delivery, and durable execution, but must be explicitly enabled and must pass the validation gates below before becoming part of the public contract.
|
||||
|
||||
## Safety Requirements
|
||||
|
||||
### Threat model
|
||||
|
||||
The design must account for:
|
||||
|
||||
- spoofed channel-native identities,
|
||||
- stolen or replayed link challenges,
|
||||
- cross-tenant or cross-confidentiality data leakage,
|
||||
- unsolicited proactive messages,
|
||||
- malicious payloads persisted for replay,
|
||||
- denial-of-service through fan-out or retry storms, and
|
||||
- privacy leakage through logs, metrics, or support tooling.
|
||||
|
||||
Required mitigations include verified identity claims where available, signed and expiring link challenges, explicit user consent, per-channel capability checks, default-deny policy options, tenant partitioning, and uninformative denial messages on shared channels.
|
||||
|
||||
### Idempotency and replay
|
||||
|
||||
Exactly-once delivery is not a realistic guarantee. The design must provide:
|
||||
|
||||
- stable run, continuation, and delivery-attempt identifiers,
|
||||
- channel-level idempotency keys where protocols support them,
|
||||
- bounded retry with jitter and explicit terminal states,
|
||||
- replay windows and expiration,
|
||||
- duplicate suppression for persisted attempts, and
|
||||
- clear semantics for "delivered", "accepted by platform", and "observed by user".
|
||||
|
||||
### Storage
|
||||
|
||||
Enhancement storage must stay distinct from v1 `AgentSession` history and workflow checkpoints unless an implementation deliberately backs them with the same physical store.
|
||||
|
||||
Stored data should be schema-versioned, minimized, encrypted or otherwise protected as appropriate, and partitioned by tenant/project. Link records, continuation records, active-channel state, delivery attempts, dead letters, and serialized payloads need independent TTL and deletion policies.
|
||||
|
||||
### Observability and support
|
||||
|
||||
The design must include structured logs, traces, and metrics for link attempts, authorization decisions, delivery scheduling, retries, replay, and dead-letter outcomes. Logs must avoid message content and sensitive identity claims by default. Operators need a way to inspect, revoke, replay, or purge stuck records safely.
|
||||
|
||||
## Validation Gates
|
||||
|
||||
Before these enhancements are accepted:
|
||||
|
||||
- A reviewed threat model covers identity linking, authorization, non-originating delivery, multicast, and replay.
|
||||
- Cross-channel linking tests prove a verified identity can link two channels and that unlink/deny paths do not leak information.
|
||||
- Authorization tests cover native-id allowlists, verified-claim allowlists, default-deny behavior, and misconfiguration failures.
|
||||
- Delivery tests cover originating-only, specific-channel, active-channel, selected-channel, and all-linked routing.
|
||||
- Background/continuation tests cover polling fallback, cancellation or expiration, process restart, retry, and dead-letter behavior.
|
||||
- Codec tests prove payloads are versioned, redacted where needed, backward compatible, and rejected safely when unknown.
|
||||
- Multicast tests prove fan-out is bounded, independently retried, and idempotent per destination.
|
||||
- Observability tests or manual validation prove support operators can correlate a request to delivery attempts without exposing sensitive content.
|
||||
|
||||
## Relationship to ADR-0027
|
||||
|
||||
ADR-0027 remains valid without any of these enhancements. This ADR extends the hosting model only after the safety, storage, and support requirements above are satisfied.
|
||||
@@ -0,0 +1,320 @@
|
||||
---
|
||||
status: proposed
|
||||
contact: eavanvalkenburg
|
||||
date: 2026-06-11
|
||||
deciders: eavanvalkenburg
|
||||
---
|
||||
|
||||
# Python hosting core and pluggable channels
|
||||
|
||||
## Scope
|
||||
|
||||
This specification is the Python implementation plan for [ADR-0027](../decisions/0027-hosting-channels.md). It documents the simplified v1 host/channel contract only.
|
||||
|
||||
The v1 contract is:
|
||||
|
||||
- `AgentFrameworkHost` owns one Starlette app, one hostable target, and one or more channels.
|
||||
- A hostable target is either a `SupportsAgentRun`-compatible agent or a `Workflow`.
|
||||
- Channels contribute routes, middleware, commands, and lifecycle callbacks.
|
||||
- Channels parse protocol-native input into `ChannelRequest`.
|
||||
- Channels render their own originating response.
|
||||
- Session continuity is explicit: a channel supplies `ChannelSession(isolation_key=...)`, and the host resolves/caches an `AgentSession` for that key.
|
||||
- The host invokes `ChannelRunHook` and `ChannelResponseHook`; channels provide hook configuration and protocol context.
|
||||
|
||||
The host does not link identities, route responses to other channels, run background continuations, or multicast in v1. Those enhancements are tracked in [ADR-0028](../decisions/0028-hosting-linking-multicast-enhancements.md).
|
||||
|
||||
## Goals
|
||||
|
||||
- Let an app expose one agent or workflow on multiple protocols without handwritten Starlette composition.
|
||||
- Keep protocol parsing and response formatting inside channel packages.
|
||||
- Provide one session-resolution path shared by all channels.
|
||||
- Keep the channel authoring surface small enough for new channels to implement.
|
||||
- Preserve full-fidelity agent and workflow results until a channel decides how to render them.
|
||||
|
||||
## Non-goals for v1
|
||||
|
||||
The following are removed from the v1 implementation pass:
|
||||
|
||||
- `IdentityLinker`, `IdentityAllowlist`, `AuthPolicy`, and `LinkPolicy`
|
||||
- `ResponseTarget`, active-channel routing, `all_linked`, fan-out, and multicast
|
||||
- `ChannelPush` and `ChannelPushCodec`
|
||||
- `DurableTaskRunner`, `InProcessTaskRunner`, and `RetryPolicy`
|
||||
- continuation tokens and background delivery
|
||||
- confidentiality tiers
|
||||
- `agent-framework-hosting-entra`
|
||||
- `local_identity_link`
|
||||
|
||||
These are follow-up design topics, not hidden requirements of the v1 host.
|
||||
|
||||
## Packages
|
||||
|
||||
| Package | Import surface | Contents |
|
||||
|---|---|---|
|
||||
| `agent-framework-hosting` | `agent_framework_hosting` | `AgentFrameworkHost`, channel protocols, key request/result types, hooks, `reset_session`, state-path helpers. |
|
||||
| `agent-framework-hosting-responses` | `agent_framework_hosting_responses` | `ResponsesChannel`. |
|
||||
| `agent-framework-hosting-invocations` | `agent_framework_hosting_invocations` | `InvocationsChannel`. |
|
||||
| `agent-framework-hosting-telegram` | `agent_framework_hosting_telegram` | `TelegramChannel` and Telegram command helpers. |
|
||||
| `agent-framework-hosting-activity-protocol` | `agent_framework_hosting_activity_protocol` | `ActivityProtocolChannel` for Activity Protocol over Azure Bot Service. |
|
||||
| `agent-framework-hosting-discord` | `agent_framework_hosting_discord` | `DiscordChannel` and Discord command/interaction helpers. |
|
||||
| `agent-framework-foundry-hosting` | `agent_framework.foundry_hosting` | Foundry isolation middleware and Foundry-backed hosting helpers usable with the v1 host. |
|
||||
|
||||
Channel packages may depend on their native SDKs. The core hosting package should not depend on channel SDKs or on top-level legacy protocol hosts.
|
||||
|
||||
## Key Types
|
||||
|
||||
### `AgentFrameworkHost`
|
||||
|
||||
The host constructor accepts:
|
||||
|
||||
- `target`: one `SupportsAgentRun`-compatible object or one `Workflow`
|
||||
- `channels`: one or more `Channel` instances
|
||||
- optional Starlette middleware
|
||||
- optional `state_dir`
|
||||
- optional workflow `checkpoint_location`
|
||||
|
||||
The host exposes:
|
||||
|
||||
- `app`: the canonical Starlette ASGI application
|
||||
- `serve(...)`: a convenience wrapper for local serving
|
||||
- `reset_session(isolation_key: str)`: rotate the cached `AgentSession` for a host-tracked conversation
|
||||
|
||||
`state_dir` is narrowed to v1 host-owned local files only:
|
||||
|
||||
- session aliases (`isolation_key` to current `AgentSession` id), and
|
||||
- workflow checkpoint paths when the app chooses the host-provided file layout.
|
||||
|
||||
It is not a store for identity links, continuations, active-channel state, delivery attempts, or multicast payloads.
|
||||
|
||||
Externally supplied isolation keys are trusted only after the channel or host middleware has authenticated and authorized the caller. The host uses `isolation_key` as a partition key; the string itself is not proof of identity or ownership.
|
||||
|
||||
### `Channel`
|
||||
|
||||
A channel implements a small protocol:
|
||||
|
||||
- declare a stable channel id/name,
|
||||
- contribute routes, middleware, commands, and lifecycle callbacks,
|
||||
- parse inbound protocol data into `ChannelRequest`,
|
||||
- call the host through `ChannelContext.run(...)` or `ChannelContext.run_stream(...)`, and
|
||||
- serialize the returned result to the originating protocol response.
|
||||
|
||||
Channels own protocol authentication, signature validation, native command registration, and protocol-specific error bodies.
|
||||
|
||||
### `ChannelContribution`
|
||||
|
||||
`ChannelContribution` is the channel's host-facing contribution:
|
||||
|
||||
- Starlette routes and optional middleware,
|
||||
- native command descriptors,
|
||||
- startup and shutdown callbacks, and
|
||||
- any channel-local metadata needed by the package.
|
||||
|
||||
The host aggregates contributions but does not interpret protocol payloads.
|
||||
|
||||
### `ChannelRequest`
|
||||
|
||||
`ChannelRequest` is the host-neutral request envelope produced by a channel. It carries:
|
||||
|
||||
- target input,
|
||||
- optional `ChannelSession`,
|
||||
- optional `ChannelIdentity`,
|
||||
- options and attributes produced by the channel, and
|
||||
- request metadata useful to hooks and context providers.
|
||||
|
||||
The host may pass attributes through to context providers and middleware. Channels should treat attributes as a documented extension bag, not as a cross-channel delivery contract.
|
||||
|
||||
### `ChannelSession`
|
||||
|
||||
`ChannelSession(isolation_key=...)` is the only v1 session-continuity mechanism.
|
||||
|
||||
When a request contains an isolation key:
|
||||
|
||||
1. The host looks up or creates the cached `AgentSession` for that key.
|
||||
2. The target runs with that `AgentSession` when the target is an agent.
|
||||
3. `reset_session(isolation_key)` rotates the alias so the next request starts a new conversation.
|
||||
|
||||
If two channels produce the same isolation key on the same host, they share the same cached session. If they produce different keys, they do not share session state.
|
||||
|
||||
### `ChannelIdentity`
|
||||
|
||||
`ChannelIdentity` is optional request metadata such as channel id, native user id, tenant id, claims, or display attributes.
|
||||
|
||||
In v1, `ChannelIdentity` does not link channels, authorize callers, select delivery destinations, or imply that two identities should share an `AgentSession`. A channel that wants shared history must still produce the same `ChannelSession.isolation_key`.
|
||||
|
||||
### Hooks
|
||||
|
||||
Hooks are optional and channel-owned:
|
||||
|
||||
- `ChannelRunHook`: runs after channel parsing and before host invocation; returns the `ChannelRequest` to execute.
|
||||
- `ChannelResponseHook`: runs after target completion and before the originating channel renders a one-shot response.
|
||||
- `ChannelStreamUpdateHook`: the host applies it to streamed updates before the originating channel serializes the stream.
|
||||
|
||||
Common uses include adapting chat text into workflow inputs, enforcing deployment-specific options, flattening rich output for text-only protocols, or filtering streamed updates for a protocol. Stream update hooks are update-only; they do not automatically sanitize `get_final_response()` output. Channels choose their response transport from the parsed protocol request before invoking run hooks.
|
||||
|
||||
### `HostedRunResult`
|
||||
|
||||
`HostedRunResult[T]` wraps the target's full-fidelity result plus the resolved `AgentSession | None`.
|
||||
|
||||
- Agent targets produce `HostedRunResult[AgentResponse]`.
|
||||
- Workflow targets produce `HostedRunResult[WorkflowRunResult]`.
|
||||
|
||||
The host does not flatten, filter, or translate the result. Each channel decides how much of the result its protocol can carry.
|
||||
|
||||
## Host Behavior
|
||||
|
||||
1. `AgentFrameworkHost` builds one Starlette app and asks each channel for its contribution.
|
||||
2. A channel route receives a protocol-native request.
|
||||
3. The channel validates/parses the native payload and creates `ChannelRequest`.
|
||||
4. The channel passes the request, optional `ChannelRunHook`, and protocol-native context to the host.
|
||||
5. The host invokes `ChannelRunHook`, if configured, and receives the prepared request.
|
||||
6. The host resolves an `AgentSession` from `ChannelSession.isolation_key` when present.
|
||||
7. The host invokes the agent or workflow target.
|
||||
8. The host wraps the result in `HostedRunResult` or the streaming equivalent.
|
||||
9. The host invokes `ChannelResponseHook`, if configured, for non-streaming/final response shaping.
|
||||
10. The host applies stream update hooks while the channel consumes streams; the channel renders the originating protocol response.
|
||||
|
||||
There is no host-level route from one channel's request to another channel's response in v1.
|
||||
|
||||
## Workflow Checkpoints
|
||||
|
||||
Workflow checkpointing is explicit. Apps either configure checkpoint storage on the workflow itself or pass a `checkpoint_location` to the host so the workflow dispatch path can use the intended file location.
|
||||
|
||||
`state_dir` may provide a conventional location for workflow checkpoint files, but checkpointing is still opt-in and separate from agent session history. Checkpoints are workflow-runtime state, not channel state and not identity-link state.
|
||||
|
||||
## Foundry Isolation Middleware
|
||||
|
||||
V1 keeps Foundry isolation as middleware rather than as a channel-linking feature.
|
||||
|
||||
The middleware is installed only when the Foundry hosting environment flag is present. In that environment it reads Foundry-provided isolation values at the trusted hosting boundary, exposes them as read-only request context for Foundry-aware history or memory providers, and rejects unsafe session resumes when the live isolation context does not match persisted session context. Outside Foundry, raw isolation headers are ignored unless an app supplies its own trusted middleware.
|
||||
|
||||
This middleware does not create cross-channel identity links and does not authorize non-Foundry channels.
|
||||
|
||||
## Current Channels
|
||||
|
||||
### Responses
|
||||
|
||||
`ResponsesChannel` exposes the OpenAI-compatible Responses API shape. It maps request body fields such as input, options, and conversation identifiers into `ChannelRequest`, and it renders Responses-compatible one-shot or streaming responses.
|
||||
|
||||
Responses session continuity uses a channel-selected `isolation_key`, commonly derived from a response/conversation id, caller-provided session id, Foundry isolation context, or deployment-specific request metadata.
|
||||
|
||||
### Invocations
|
||||
|
||||
`InvocationsChannel` exposes an invocation endpoint for server-side callers and tools. It maps the request body into `ChannelRequest` and renders the invocation result on the same HTTP response.
|
||||
|
||||
Invocations is useful for typed workflow inputs because a `ChannelRunHook` can translate the request body into the workflow's expected input type.
|
||||
|
||||
### Telegram
|
||||
|
||||
`TelegramChannel` supports webhook or polling transport, native command registration, and message rendering back to the originating Telegram chat.
|
||||
|
||||
The channel chooses a default `isolation_key` from Telegram-native data such as chat id, user id, or a configured user/chat scope. A `/new` or equivalent command may call `reset_session` for that isolation key.
|
||||
|
||||
### Activity Protocol
|
||||
|
||||
`ActivityChannel` supports Activity Protocol requests, typically through Azure Bot Service for Teams, Web Chat, and other Bot Framework-fronted surfaces.
|
||||
|
||||
The channel maps incoming `Activity` objects to `ChannelRequest` and renders a reply activity to the originating conversation. Proactive Activity delivery, active-channel routing, and all-linked fan-out are not v1 host semantics.
|
||||
|
||||
### Discord
|
||||
|
||||
`DiscordChannel` supports Discord messages, slash commands, and interactions as channel-native input.
|
||||
|
||||
The channel maps Discord-native user, guild, channel, thread, and interaction data into `ChannelRequest` metadata and a configured `ChannelSession.isolation_key`. It renders the result to the originating Discord response path.
|
||||
|
||||
## High-level Samples
|
||||
|
||||
### One agent on Responses
|
||||
|
||||
```python
|
||||
host = AgentFrameworkHost(
|
||||
target=agent,
|
||||
channels=[ResponsesChannel()],
|
||||
)
|
||||
|
||||
app = host.app
|
||||
```
|
||||
|
||||
### One agent on multiple channels
|
||||
|
||||
```python
|
||||
host = AgentFrameworkHost(
|
||||
target=agent,
|
||||
channels=[
|
||||
ResponsesChannel(),
|
||||
InvocationsChannel(),
|
||||
TelegramChannel(bot_token=os.environ["TELEGRAM_BOT_TOKEN"]),
|
||||
],
|
||||
)
|
||||
|
||||
host.serve(host="localhost", port=8000)
|
||||
```
|
||||
|
||||
The host owns one Starlette app. Each channel contributes its own routes and renders its own response.
|
||||
|
||||
### Adapting a request before execution
|
||||
|
||||
```python
|
||||
from dataclasses import replace
|
||||
|
||||
|
||||
def enforce_options(request: ChannelRequest) -> ChannelRequest:
|
||||
options = dict(request.options or {})
|
||||
options["temperature"] = 0
|
||||
return replace(request, options=options)
|
||||
|
||||
|
||||
host = AgentFrameworkHost(
|
||||
target=agent,
|
||||
channels=[ResponsesChannel(run_hook=enforce_options)],
|
||||
)
|
||||
```
|
||||
|
||||
### Workflow with explicit checkpoints
|
||||
|
||||
```python
|
||||
host = AgentFrameworkHost(
|
||||
target=workflow,
|
||||
channels=[InvocationsChannel(run_hook=adapt_to_workflow_input)],
|
||||
checkpoint_location=Path("./.af-hosting/workflow_checkpoints"),
|
||||
)
|
||||
```
|
||||
|
||||
The hook adapts channel-native input to the workflow's typed input. Checkpoints use the explicit workflow checkpoint location, not identity-link or delivery storage.
|
||||
|
||||
### Message channel reset command
|
||||
|
||||
```python
|
||||
async def new_chat(context):
|
||||
if context.request.session is not None:
|
||||
await context.host.reset_session(context.request.session.isolation_key)
|
||||
await context.reply("Started a new conversation.")
|
||||
```
|
||||
|
||||
Telegram, Activity Protocol, and Discord can expose equivalent native commands when their protocols support them.
|
||||
|
||||
## Follow-up Enhancements
|
||||
|
||||
See [ADR-0028](../decisions/0028-hosting-linking-multicast-enhancements.md) for the deferred design covering:
|
||||
|
||||
- cross-channel identity linking,
|
||||
- authorization and allowlists,
|
||||
- non-originating response delivery,
|
||||
- active-channel routing,
|
||||
- multicast and all-linked delivery,
|
||||
- background runs and continuation tokens,
|
||||
- durable delivery runners,
|
||||
- retry/replay semantics, and
|
||||
- payload serialization.
|
||||
|
||||
Those enhancements must layer on top of this v1 contract without requiring v1 users to adopt them.
|
||||
|
||||
## Validation Gates
|
||||
|
||||
The Python implementation should be considered complete when:
|
||||
|
||||
- a sample uses one `AgentFrameworkHost` with multiple channels and no manual Starlette route composition,
|
||||
- each current channel has contract tests for route contribution, lifecycle, request parsing, hooks, and originating response rendering,
|
||||
- session tests prove shared `isolation_key` values share an `AgentSession` and `reset_session` rotates it,
|
||||
- workflow tests or samples use explicit `checkpoint_location`,
|
||||
- Foundry isolation middleware is covered by integration or contract tests,
|
||||
- no v1 package exposes the removed linking, multicast, durable-runner, or continuation APIs, and
|
||||
- this spec and ADR-0027 remain aligned.
|
||||
@@ -12,7 +12,7 @@
|
||||
<ItemGroup>
|
||||
<!-- Aspire.* -->
|
||||
<PackageVersion Include="Anthropic" Version="12.20.0" />
|
||||
<PackageVersion Include="Anthropic.Foundry" Version="0.5.0" />
|
||||
<PackageVersion Include="Anthropic.Foundry" Version="0.6.0" />
|
||||
<PackageVersion Include="Aspire.Hosting" Version="$(AspireAppHostSdkVersion)" />
|
||||
<PackageVersion Include="Aspire.Azure.AI.OpenAI" Version="13.0.0-preview.1.25560.3" />
|
||||
<PackageVersion Include="Aspire.Azure.AI.Inference" Version="13.1.0-preview.1.25616.3" />
|
||||
@@ -27,10 +27,10 @@
|
||||
<PackageVersion Include="Azure.AI.AgentServer.Invocations" Version="1.0.0-beta.4" />
|
||||
<PackageVersion Include="Azure.AI.AgentServer.Responses" Version="1.0.0-beta.5" />
|
||||
<PackageVersion Include="Azure.Search.Documents" Version="12.0.0" />
|
||||
<PackageVersion Include="Azure.AI.Projects" Version="2.1.0-beta.2" />
|
||||
<PackageVersion Include="Azure.AI.Projects" Version="2.1.0-beta.3" />
|
||||
<PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.2.0-beta.10" />
|
||||
<PackageVersion Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
|
||||
<PackageVersion Include="Azure.Core" Version="1.56.0" />
|
||||
<PackageVersion Include="Azure.Core" Version="1.57.0" />
|
||||
<PackageVersion Include="Azure.Identity" Version="1.21.0" />
|
||||
<PackageVersion Include="DotNetEnv" Version="3.1.1" />
|
||||
<PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.5.0" />
|
||||
@@ -45,7 +45,7 @@
|
||||
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.8" />
|
||||
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="6.0.0" />
|
||||
<PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.5" />
|
||||
<PackageVersion Include="System.ClientModel" Version="1.12.0" />
|
||||
<PackageVersion Include="System.ClientModel" Version="1.13.0" />
|
||||
<PackageVersion Include="System.CodeDom" Version="10.0.0" />
|
||||
<PackageVersion Include="System.Collections.Immutable" Version="10.0.1" />
|
||||
<PackageVersion Include="System.CommandLine" Version="2.0.0-rc.2.25502.107" />
|
||||
|
||||
@@ -411,6 +411,7 @@
|
||||
<Folder Name="/Samples/05-end-to-end/Evaluation/">
|
||||
<Project Path="samples/05-end-to-end/Evaluation/Evaluation_ConversationSplits/Evaluation_ConversationSplits.csproj" />
|
||||
<Project Path="samples/05-end-to-end/Evaluation/Evaluation_FoundryQuality/Evaluation_FoundryQuality.csproj" />
|
||||
<Project Path="samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Evaluation_FoundryRubric.csproj" />
|
||||
<Project Path="samples/05-end-to-end/Evaluation/Evaluation_MixedProviders/Evaluation_MixedProviders.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/05-end-to-end/A2AClientServer/">
|
||||
|
||||
@@ -26,8 +26,8 @@ internal static class GetStartedSamples
|
||||
{
|
||||
Name = "01_hello_agent",
|
||||
ProjectPath = "samples/01-get-started/01_hello_agent",
|
||||
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
|
||||
RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["FOUNDRY_MODEL"],
|
||||
ExpectedOutputDescription =
|
||||
[
|
||||
"The output should contain a joke about a pirate.",
|
||||
@@ -40,8 +40,8 @@ internal static class GetStartedSamples
|
||||
{
|
||||
Name = "02_add_tools",
|
||||
ProjectPath = "samples/01-get-started/02_add_tools",
|
||||
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
|
||||
RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["FOUNDRY_MODEL"],
|
||||
MustContain = [],
|
||||
ExpectedOutputDescription =
|
||||
[
|
||||
@@ -56,8 +56,8 @@ internal static class GetStartedSamples
|
||||
{
|
||||
Name = "03_multi_turn",
|
||||
ProjectPath = "samples/01-get-started/03_multi_turn",
|
||||
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
|
||||
RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["FOUNDRY_MODEL"],
|
||||
ExpectedOutputDescription =
|
||||
[
|
||||
"The output should contain a joke about a pirate.",
|
||||
@@ -71,8 +71,8 @@ internal static class GetStartedSamples
|
||||
{
|
||||
Name = "04_memory",
|
||||
ProjectPath = "samples/01-get-started/04_memory",
|
||||
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
|
||||
RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["FOUNDRY_MODEL"],
|
||||
MustContain =
|
||||
[
|
||||
">> Use session with blank memory",
|
||||
@@ -97,8 +97,8 @@ internal static class GetStartedSamples
|
||||
{
|
||||
Name = "06_host_your_agent",
|
||||
ProjectPath = "samples/01-get-started/06_host_your_agent",
|
||||
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
|
||||
RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT"],
|
||||
OptionalEnvironmentVariables = ["FOUNDRY_MODEL"],
|
||||
SkipReason = "Requires Azure Functions Core Tools runtime and starts a web server.",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -9,13 +9,12 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,23 +1,19 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample shows how to create and use a simple AI agent with Azure OpenAI as the backend.
|
||||
// This sample shows how to create and use a simple AI agent with AIProjectClient as the backend.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Chat;
|
||||
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
var model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIAgent agent = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetChatClient(deploymentName)
|
||||
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
|
||||
AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.AsAIAgent(model: model, instructions: "You are good at telling jokes.", name: "Joker");
|
||||
|
||||
// Invoke the agent and output the text result.
|
||||
Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
|
||||
|
||||
@@ -9,13 +9,12 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -1,31 +1,27 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates how to use a ChatClientAgent with function tools.
|
||||
// It shows both non-streaming and streaming agent interactions using menu-related tools.
|
||||
// This sample demonstrates how to use an AIProjectClient agent with function tools.
|
||||
// It shows both non-streaming and streaming agent interactions using weather tools.
|
||||
|
||||
using System.ComponentModel;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
using OpenAI.Chat;
|
||||
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
var model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini";
|
||||
|
||||
[Description("Get the weather for a given location.")]
|
||||
static string GetWeather([Description("The location to get the weather for.")] string location)
|
||||
=> $"The weather in {location} is cloudy with a high of 15°C.";
|
||||
|
||||
// Create the chat client and agent, and provide the function tool to the agent.
|
||||
// Create the agent and provide the function tool to the agent.
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIAgent agent = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetChatClient(deploymentName)
|
||||
.AsAIAgent(instructions: "You are a helpful assistant", tools: [AIFunctionFactory.Create(GetWeather)]);
|
||||
AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.AsAIAgent(model: model, instructions: "You are a helpful assistant", tools: [AIFunctionFactory.Create(GetWeather)]);
|
||||
|
||||
// Non-streaming agent interaction with function tools.
|
||||
Console.WriteLine(await agent.RunAsync("What is the weather like in Amsterdam?"));
|
||||
|
||||
@@ -9,13 +9,12 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -2,22 +2,18 @@
|
||||
|
||||
// This sample shows how to create and use a simple AI agent with a multi-turn conversation.
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Chat;
|
||||
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
var model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIAgent agent = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetChatClient(deploymentName)
|
||||
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
|
||||
AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.AsAIAgent(model: model, instructions: "You are good at telling jokes.", name: "Joker");
|
||||
|
||||
// Invoke the agent with a multi-turn conversation, where the context is preserved in the session object.
|
||||
AgentSession session = await agent.CreateSessionAsync();
|
||||
|
||||
@@ -9,13 +9,12 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -8,23 +8,26 @@
|
||||
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
using OpenAI.Chat;
|
||||
using SampleApp;
|
||||
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
var model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
ChatClient chatClient = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetChatClient(deploymentName);
|
||||
var projectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// Get the underlying IChatClient to use for the memory component.
|
||||
// The memory provider needs direct IChatClient access for structured extraction.
|
||||
IChatClient chatClient = projectClient
|
||||
.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new() { ModelId = model } })
|
||||
.GetService<IChatClient>()
|
||||
?? throw new InvalidOperationException("Could not retrieve IChatClient from AIProjectClient agent.");
|
||||
|
||||
// Create the agent and provide a factory to add our custom memory component to
|
||||
// all sessions created by the agent. Here each new memory component will have its own
|
||||
@@ -36,7 +39,7 @@ ChatClient chatClient = new AzureOpenAIClient(
|
||||
AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions()
|
||||
{
|
||||
ChatOptions = new() { Instructions = "You are a friendly assistant. Always address the user by their name." },
|
||||
AIContextProviders = [new UserInfoMemory(chatClient.AsIChatClient())]
|
||||
AIContextProviders = [new UserInfoMemory(chatClient)]
|
||||
});
|
||||
|
||||
// Create a new session for the conversation.
|
||||
|
||||
@@ -21,11 +21,10 @@
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.Hosting.AzureFunctions\Microsoft.Agents.AI.Hosting.AzureFunctions.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -4,36 +4,32 @@
|
||||
//
|
||||
// Prerequisites:
|
||||
// - Azure Functions Core Tools
|
||||
// - Azure OpenAI resource
|
||||
// - Foundry project endpoint and credentials
|
||||
//
|
||||
// Environment variables:
|
||||
// AZURE_OPENAI_ENDPOINT
|
||||
// AZURE_OPENAI_DEPLOYMENT_NAME (defaults to "gpt-5.4-mini")
|
||||
// FOUNDRY_PROJECT_ENDPOINT
|
||||
// FOUNDRY_MODEL (defaults to "gpt-5.4-mini")
|
||||
//
|
||||
// Run with: func start
|
||||
// Then call: POST http://localhost:7071/api/agents/HostedAgent/run
|
||||
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Hosting.AzureFunctions;
|
||||
using Microsoft.Azure.Functions.Worker.Builder;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using OpenAI.Chat;
|
||||
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT")
|
||||
?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
var model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini";
|
||||
|
||||
// Set up an AI agent following the standard Microsoft Agent Framework pattern.
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIAgent agent = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.GetChatClient(deploymentName)
|
||||
.AsAIAgent(
|
||||
instructions: "You are a helpful assistant hosted in Azure Functions.",
|
||||
name: "HostedAgent");
|
||||
AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.AsAIAgent(model: model, instructions: "You are a helpful assistant hosted in Azure Functions.", name: "HostedAgent");
|
||||
|
||||
// Configure the function app to host the AI agent.
|
||||
// This will automatically generate HTTP API endpoints for the agent.
|
||||
|
||||
+3
-4
@@ -56,14 +56,13 @@ try
|
||||
// Inspect memory search results if available in raw response items.
|
||||
foreach (var message in response.Messages)
|
||||
{
|
||||
if (message.RawRepresentation is MemorySearchToolCallResponseItem memorySearchResult)
|
||||
if (message.RawRepresentation is MemorySearchToolCall memorySearchResult)
|
||||
{
|
||||
Console.WriteLine($"Memory Search Status: {memorySearchResult.Status}");
|
||||
Console.WriteLine($"Memory Search Results Count: {memorySearchResult.Results.Count}");
|
||||
Console.WriteLine($"Memory Search Results Count: {memorySearchResult.Memories.Count}");
|
||||
|
||||
foreach (var result in memorySearchResult.Results)
|
||||
foreach (var memoryItem in memorySearchResult.Memories)
|
||||
{
|
||||
var memoryItem = result.MemoryItem;
|
||||
Console.WriteLine($" - Memory ID: {memoryItem.MemoryId}");
|
||||
Console.WriteLine($" Scope: {memoryItem.Scope}");
|
||||
Console.WriteLine($" Content: {memoryItem.Content}");
|
||||
|
||||
@@ -31,3 +31,4 @@ dotnet run --project .\Evaluation_ExpectedOutputs
|
||||
|
||||
- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in and custom checks
|
||||
- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators
|
||||
- [Evaluation_FoundryRubric](../../../05-end-to-end/Evaluation/Evaluation_FoundryRubric/) — Rubric (adaptive) evaluators with per-dimension scores
|
||||
|
||||
@@ -26,4 +26,5 @@ dotnet run --project .\Evaluation_Multimodal
|
||||
|
||||
- [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in checks and `agent.EvaluateAsync()`
|
||||
- [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators
|
||||
- [Evaluation_FoundryRubric](../../../05-end-to-end/Evaluation/Evaluation_FoundryRubric/) — Rubric (adaptive) evaluators with per-dimension scores
|
||||
- [Evaluation_ConversationSplits](../../../05-end-to-end/Evaluation/Evaluation_ConversationSplits/) — Multi-turn conversation split strategies
|
||||
|
||||
+6
-6
@@ -56,9 +56,9 @@ AIAgent webSearchAgent =
|
||||
OpenTelemetrySourceName = TracingSourceName,
|
||||
DisableTodoProvider = true,
|
||||
DisableAgentModeProvider = true,
|
||||
DisableFileMemory = true, // If enabled, this would allow the agent to store memories as files in a directory associated with the current session
|
||||
DisableFileAccess = true, // If enabled, this would allow the agent to read/write files in a working directory
|
||||
DisableToolApproval = true, // If enabled, this allows don't-ask-again approval functionality.
|
||||
DisableFileMemory = true, // If enabled, this would allow the agent to store memories as files in a directory associated with the current session
|
||||
DisableFileAccess = true, // If enabled, this would allow the agent to read/write files in a working directory
|
||||
DisableToolAutoApproval = true, // If true, this disables the don't-ask-again approval functionality.
|
||||
ChatOptions = new ChatOptions
|
||||
{
|
||||
Instructions = "You are a web search assistant. When asked to find information, use the web search tool to look it up and return a concise, factual answer.",
|
||||
@@ -106,9 +106,9 @@ AIAgent parentAgent =
|
||||
OpenTelemetrySourceName = TracingSourceName,
|
||||
DisableTodoProvider = true,
|
||||
DisableAgentModeProvider = true,
|
||||
DisableFileMemory = true, // If enabled, this would allow the agent to store memories as files in a directory associated with the current session
|
||||
DisableFileAccess = true, // If enabled, this would allow the agent to read/write files in a working directory
|
||||
DisableToolApproval = true, // If enabled, this allows don't-ask-again approval functionality.
|
||||
DisableFileMemory = true, // If enabled, this would allow the agent to store memories as files in a directory associated with the current session
|
||||
DisableFileAccess = true, // If enabled, this would allow the agent to read/write files in a working directory
|
||||
DisableToolAutoApproval = true, // If true, this disables the don't-ask-again approval functionality.
|
||||
DisableWebSearch = true,
|
||||
BackgroundAgents = [webSearchAgent],
|
||||
ChatOptions = new ChatOptions
|
||||
|
||||
@@ -79,6 +79,13 @@ AIAgent agent =
|
||||
Description = "A data analyst assistant that reads, analyzes, and processes data files.",
|
||||
OpenTelemetrySourceName = TracingSourceName,
|
||||
FileAccessStore = new FileSystemAgentFileStore(Path.Combine(AppContext.BaseDirectory, "working")),
|
||||
ToolApprovalAgentOptions = new ToolApprovalAgentOptions()
|
||||
{
|
||||
// The HarnessAgent's FileAccessProvider requires approval for all file access operations.
|
||||
// Add an auto-approval rule to skip prompts for specific operations (e.g., read-only access).
|
||||
// You can also supply your own rule to implement custom approval logic.
|
||||
AutoApprovalRules = [FileAccessProvider.ReadOnlyToolsAutoApprovalRule]
|
||||
},
|
||||
DisableTodoProvider = true,
|
||||
DisableAgentModeProvider = true,
|
||||
DisableFileMemory = true, // If enabled, this would allow the agent to store memories as files in a directory associated with the current session
|
||||
|
||||
@@ -27,7 +27,7 @@ public static class Program
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
var client = new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()).GetChatClient(deploymentName).AsIChatClient();
|
||||
|
||||
Console.Write("Choose workflow type ('sequential', 'concurrent', 'handoffs', 'groupchat'): ");
|
||||
Console.Write("Choose workflow type ('sequential', 'sequential-chain-only', 'concurrent', 'handoffs', 'groupchat'): ");
|
||||
switch (Console.ReadLine())
|
||||
{
|
||||
case "sequential":
|
||||
@@ -36,6 +36,14 @@ public static class Program
|
||||
[new(ChatRole.User, "Hello, world!")]);
|
||||
break;
|
||||
|
||||
case "sequential-chain-only":
|
||||
await RunWorkflowAsync(
|
||||
AgentWorkflowBuilder.BuildSequential(
|
||||
chainOnlyAgentResponses: true,
|
||||
from lang in (string[])["French", "Spanish", "English"] select GetTranslationAgent(lang, client)),
|
||||
[new(ChatRole.User, "Hello, world!")]);
|
||||
break;
|
||||
|
||||
case "concurrent":
|
||||
await RunWorkflowAsync(
|
||||
AgentWorkflowBuilder.BuildConcurrent(from lang in (string[])["French", "Spanish", "English"] select GetTranslationAgent(lang, client)),
|
||||
|
||||
+6
-47
@@ -20,7 +20,6 @@
|
||||
#pragma warning disable AAIP001 // ProjectAgentSkills is experimental
|
||||
|
||||
using System.ClientModel;
|
||||
using System.IO.Compression;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Core;
|
||||
@@ -121,8 +120,8 @@ app.Run();
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// Downloads each named skill from Foundry and extracts the ZIP archive into a
|
||||
// separate subdirectory under the target directory.
|
||||
// Downloads each named skill from Foundry into a separate subdirectory under the target directory.
|
||||
// GetSkillContentAsync downloads the skill package and unzips it into the destination directory.
|
||||
static async Task DownloadSkillsAsync(ProjectAgentSkills skillsClient, string[] skillNames, string targetDir)
|
||||
{
|
||||
if (Directory.Exists(targetDir))
|
||||
@@ -135,56 +134,16 @@ static async Task DownloadSkillsAsync(ProjectAgentSkills skillsClient, string[]
|
||||
foreach (string name in skillNames)
|
||||
{
|
||||
Console.WriteLine($"Downloading skill '{name}' from Foundry...");
|
||||
BinaryData zipData = await skillsClient.DownloadSkillAsync(name);
|
||||
|
||||
string skillDir = Path.Combine(targetDir, name);
|
||||
Directory.CreateDirectory(skillDir);
|
||||
|
||||
using var zipStream = zipData.ToStream();
|
||||
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read);
|
||||
SafeExtractZip(archive, skillDir);
|
||||
await skillsClient.GetSkillContentAsync(name, skillDir);
|
||||
|
||||
if (!File.Exists(Path.Combine(skillDir, "SKILL.md")))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Downloaded archive for '{name}' did not contain a SKILL.md at the root.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extracts a ZIP archive into a destination directory, rejecting entries that would
|
||||
// escape the target path (zip-slip guard).
|
||||
static void SafeExtractZip(ZipArchive archive, string destinationDir)
|
||||
{
|
||||
string destRoot = Path.GetFullPath(destinationDir);
|
||||
string destRootWithSep = Path.EndsInDirectorySeparator(destRoot)
|
||||
? destRoot
|
||||
: destRoot + Path.DirectorySeparatorChar;
|
||||
|
||||
// Use ordinal comparison on Unix (case-sensitive FS) and ordinal-ignore-case on Windows.
|
||||
var comparison = OperatingSystem.IsWindows()
|
||||
? StringComparison.OrdinalIgnoreCase
|
||||
: StringComparison.Ordinal;
|
||||
|
||||
foreach (ZipArchiveEntry entry in archive.Entries)
|
||||
{
|
||||
string entryPath = Path.GetFullPath(Path.Combine(destRoot, entry.FullName));
|
||||
if (!entryPath.StartsWith(destRootWithSep, comparison)
|
||||
&& !string.Equals(entryPath, destRoot, comparison))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Refusing to extract unsafe path '{entry.FullName}' outside of '{destRoot}'.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(entry.Name))
|
||||
{
|
||||
// Directory entry — ensure it exists.
|
||||
Directory.CreateDirectory(entryPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(entryPath)!);
|
||||
entry.ExtractToFile(entryPath, overwrite: true);
|
||||
$"Downloaded skill '{name}' did not contain a SKILL.md at the root.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,8 +170,8 @@ static async Task EnsureSkillsProvisionedAsync(ProjectAgentSkills skillsClient,
|
||||
catch (ClientResultException ex) when (ex.Status == 404)
|
||||
{
|
||||
Console.WriteLine($"Provisioning skill '{name}' from {skillPath}...");
|
||||
AgentsSkill imported = await skillsClient.CreateSkillFromPackageAsync(skillPath);
|
||||
Console.WriteLine($" Imported skill '{imported.Name}' (id={imported.SkillId}, has_blob={imported.HasBlob}).");
|
||||
AgentsSkill imported = (await skillsClient.CreateSkillVersionFromFilesAsync(name, skillPath)).Value;
|
||||
Console.WriteLine($" Imported skill '{imported.Name}' (id={imported.Id}, version={imported.LatestVersion}).");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ The agent is hosted using the [Agent Framework](https://github.com/microsoft/age
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
### Required RBAC
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ This sample is the Azure AI Search counterpart to `Hosted-TextRag`. Where `Hoste
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- An Azure AI Search service ([create one](https://learn.microsoft.com/azure/search/search-create-service-portal))
|
||||
- **A pre-provisioned search index** with the schema and content described in the next section
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ A simple general-purpose AI assistant hosted as a Foundry Hosted Agent using the
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
@@ -16,7 +16,7 @@ Copy the template and fill in your project endpoint:
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint:
|
||||
Edit `.env` and set your Foundry project endpoint:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
// indirect prompt injection in an uploaded file.
|
||||
//
|
||||
// Required environment variables:
|
||||
// FOUNDRY_PROJECT_ENDPOINT - Azure AI Foundry project endpoint
|
||||
// FOUNDRY_PROJECT_ENDPOINT - Foundry project endpoint
|
||||
// FOUNDRY_MODEL - Model deployment name (default: gpt-4o)
|
||||
//
|
||||
// Optional:
|
||||
|
||||
@@ -43,7 +43,7 @@ The end-to-end alpha-SDK round trip (client uploads via `AgentSessionFiles.Uploa
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ This is the **Foundry hosting** pattern — the agent's behavior is configured i
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a **registered agent** (created via Foundry UI, CLI, or API)
|
||||
- A Foundry project with a **registered agent** (created via Foundry UI, CLI, or API)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
@@ -18,7 +18,7 @@ Copy the template and fill in your project endpoint:
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint:
|
||||
Edit `.env` and set your Foundry project endpoint:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
+2
-2
@@ -21,7 +21,7 @@ sample for the `agent-framework-local-codeact` package.
|
||||
- Python 3 available on `PATH` (used by `LocalCodeActProvider` to execute the
|
||||
embedded runner and validator). Override with the `LOCAL_CODEACT_PYTHON`
|
||||
environment variable if you need a specific interpreter path.
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
@@ -32,7 +32,7 @@ Copy the template and fill in your project endpoint:
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint:
|
||||
Edit `.env` and set your Foundry project endpoint:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
@@ -7,7 +7,7 @@ The agent specializes in finding hotels in Seattle, with a `GetAvailableHotels`
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
@@ -18,7 +18,7 @@ Copy the template and fill in your project endpoint:
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint:
|
||||
Edit `.env` and set your Foundry project endpoint:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
@@ -19,7 +19,7 @@ A hosted agent demonstrating **two layers of MCP (Model Context Protocol) tool i
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ This sample exists to demonstrate two things together:
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with at least one chat model deployment and one embedding model deployment
|
||||
- A Foundry project with at least one chat model deployment and one embedding model deployment
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ Foundry injects `APPLICATIONINSIGHTS_CONNECTION_STRING` when the agent runs in t
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
@@ -36,7 +36,7 @@ Foundry injects `APPLICATIONINSIGHTS_CONNECTION_STRING` when the agent runs in t
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint:
|
||||
Edit `.env` and set your Foundry project endpoint:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
@@ -7,7 +7,7 @@ This sample demonstrates how to add knowledge grounding to a hosted agent withou
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
@@ -18,7 +18,7 @@ Copy the template and fill in your project endpoint:
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint:
|
||||
Edit `.env` and set your Foundry project endpoint:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Azure AI Foundry project endpoint (auto-injected in hosted containers).
|
||||
# Foundry project endpoint (auto-injected in hosted containers).
|
||||
AZURE_AI_PROJECT_ENDPOINT=https://<your-foundry-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
# Model deployment name. Must exist in the Foundry project above.
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
//
|
||||
// Required environment variables:
|
||||
// AZURE_AI_PROJECT_ENDPOINT (local-dev) OR FOUNDRY_PROJECT_ENDPOINT (hosted runtime)
|
||||
// - Azure AI Foundry project endpoint. The Foundry hosted
|
||||
// - Foundry project endpoint. The Foundry hosted
|
||||
// runtime auto-injects FOUNDRY_PROJECT_ENDPOINT; locally
|
||||
// set AZURE_AI_PROJECT_ENDPOINT (the AF-repo convention).
|
||||
// TOOLBOX_NAME - Name of the Foundry Toolbox to load
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# Foundry project endpoint (auto-injected in hosted containers).
|
||||
AZURE_AI_PROJECT_ENDPOINT=https://<your-foundry-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
# Model deployment name. Must exist in the Foundry project above.
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o
|
||||
|
||||
# Name of the Foundry Toolbox you provisioned in the portal (see README.md).
|
||||
TOOLBOX_NAME=my-toolset
|
||||
|
||||
# Agent name advertised over the wire. Must be unique if running side-by-side with
|
||||
# other Hosted-* samples (e.g. Hosted-Toolbox-AuthPaths), otherwise the REPL client
|
||||
# cannot disambiguate which agent to chat with.
|
||||
AGENT_NAME=hosted-toolbox-agent
|
||||
|
||||
# Application Insights connection string (auto-injected in hosted containers; optional locally).
|
||||
# APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=...
|
||||
@@ -0,0 +1,17 @@
|
||||
# Use the official .NET 10.0 ASP.NET runtime as a parent image
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
|
||||
WORKDIR /app
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
RUN dotnet restore
|
||||
RUN dotnet publish -c Release -o /app/publish
|
||||
|
||||
# Final stage
|
||||
FROM base AS final
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/publish .
|
||||
EXPOSE 8088
|
||||
ENV ASPNETCORE_URLS=http://+:8088
|
||||
ENTRYPOINT ["dotnet", "HostedToolbox.dll"]
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# Dockerfile for contributors building from the agent-framework repository source.
|
||||
#
|
||||
# This project uses ProjectReference to the local source, which means a standard
|
||||
# multi-stage Docker build cannot resolve dependencies outside this folder.
|
||||
# Pre-publish the app targeting the container runtime and copy the output:
|
||||
#
|
||||
# dotnet publish -c Debug -f net10.0 -r linux-musl-x64 --self-contained false -o out
|
||||
# docker build -f Dockerfile.contributor -t hosted-toolbox .
|
||||
# docker run --rm -p 8088:8088 \
|
||||
# -e AGENT_NAME=hosted-toolbox-agent \
|
||||
# -e AZURE_BEARER_TOKEN=$AZURE_BEARER_TOKEN \
|
||||
# --env-file .env hosted-toolbox
|
||||
#
|
||||
# For end-users consuming the NuGet package (not ProjectReference), use the standard
|
||||
# Dockerfile which performs a full dotnet restore + publish inside the container.
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final
|
||||
WORKDIR /app
|
||||
COPY out/ .
|
||||
EXPOSE 8088
|
||||
ENV ASPNETCORE_URLS=http://+:8088
|
||||
ENTRYPOINT ["dotnet", "HostedToolbox.dll"]
|
||||
@@ -7,20 +7,19 @@
|
||||
//
|
||||
// Required environment variables:
|
||||
// FOUNDRY_PROJECT_ENDPOINT (hosted runtime) OR AZURE_AI_PROJECT_ENDPOINT (local-dev)
|
||||
// - Azure AI Foundry project endpoint. The Foundry hosted
|
||||
// - Foundry project endpoint. The Foundry hosted
|
||||
// runtime auto-injects FOUNDRY_PROJECT_ENDPOINT; locally
|
||||
// set AZURE_AI_PROJECT_ENDPOINT.
|
||||
// FOUNDRY_MODEL - Model deployment name (default: gpt-4o)
|
||||
//
|
||||
// Optional:
|
||||
// FOUNDRY_TOOLBOX_NAME - Name of the toolbox to load (default: my-toolset)
|
||||
// FOUNDRY_AGENT_TOOLSET_ENDPOINT - Foundry Toolsets proxy base URL
|
||||
// (injected automatically by Foundry platform at runtime)
|
||||
// FOUNDRY_AGENT_NAME - Client name reported to MCP server (auto-injected in hosted runtime)
|
||||
// FOUNDRY_AGENT_VERSION - Client version reported to MCP server (auto-injected in hosted runtime)
|
||||
// FOUNDRY_AGENT_TOOLSET_FEATURES - Additional Foundry-Features header flags (the mandatory
|
||||
// Toolboxes=V1Preview flag is always sent; this env var
|
||||
// appends additional flags if present).
|
||||
// FOUNDRY_MODEL (or AZURE_AI_MODEL_DEPLOYMENT_NAME)
|
||||
// - Model deployment name (default: gpt-4o)
|
||||
// TOOLBOX_NAME - Name of the toolbox to load (default: my-toolset).
|
||||
// NOTE: All FOUNDRY_* and AGENT_* env-var prefixes (other
|
||||
// than the platform-injected ones above) are reserved by the
|
||||
// Foundry container platform and rejected at agent-create.
|
||||
// Use TOOLBOX_NAME, not FOUNDRY_TOOLBOX_NAME, for the
|
||||
// sample-owned toolbox name so it survives deployment.
|
||||
//
|
||||
// The Foundry.Hosting package builds the toolbox proxy URL from FOUNDRY_PROJECT_ENDPOINT
|
||||
// per tools-integration-spec.md §2–§3.
|
||||
@@ -43,8 +42,7 @@ string endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT")
|
||||
"nor AZURE_AI_PROJECT_ENDPOINT (local-dev convention) is set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL")
|
||||
?? Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o";
|
||||
string toolboxName = Environment.GetEnvironmentVariable("FOUNDRY_TOOLBOX_NAME")
|
||||
?? Environment.GetEnvironmentVariable("TOOLBOX_NAME") ?? "my-toolset";
|
||||
string toolboxName = Environment.GetEnvironmentVariable("TOOLBOX_NAME") ?? "my-toolset";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
|
||||
@@ -1,27 +1,111 @@
|
||||
# Hosted-Toolbox
|
||||
|
||||
A hosted Foundry agent that loads tools from a Foundry Toolbox via the AF Foundry hosting bridge.
|
||||
A hosted Foundry agent that loads tools from a single Foundry Toolbox via the AF Foundry hosting bridge.
|
||||
|
||||
The agent declares one `FoundryAITool.CreateHostedMcpToolbox(name)` marker; `AddFoundryToolboxes(name)` registers a `FoundryToolboxService` that resolves the marker into the individual MCP tools the toolbox bundles, connecting to the Foundry Toolboxes MCP proxy at startup and discovering tools via `tools/list`.
|
||||
`AddFoundryToolboxes(name)` registers a `FoundryToolboxService` that connects to the Foundry Toolboxes MCP proxy at startup, discovers the toolbox's bundled tools via `tools/list`, and makes them available to the agent on every request. The agent code does nothing per request; the toolbox is baked in on the server.
|
||||
|
||||
This is the minimal toolbox intro. For a richer walkthrough where a single toolbox bundles three MCP tools each authenticated differently, see [`Hosted-Toolbox-AuthPaths/`](../Hosted-Toolbox-AuthPaths/).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A Microsoft Foundry project with a Toolbox configured.
|
||||
- Azure CLI logged in (`az login`).
|
||||
- Set environment variables:
|
||||
- `AZURE_AI_PROJECT_ENDPOINT` (local-dev) or `FOUNDRY_PROJECT_ENDPOINT` (auto-injected in hosted containers)
|
||||
- `AZURE_AI_MODEL_DEPLOYMENT_NAME` (default `gpt-4o`)
|
||||
- `TOOLBOX_NAME` (default `my-toolbox`)
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-4o`) and a Toolbox configured
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
|
||||
Copy the template and fill in your values:
|
||||
|
||||
```powershell
|
||||
Copy-Item .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env`:
|
||||
|
||||
```env
|
||||
AZURE_AI_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o
|
||||
TOOLBOX_NAME=my-toolset
|
||||
```
|
||||
|
||||
Configuration notes:
|
||||
|
||||
- `AZURE_AI_PROJECT_ENDPOINT` (local-dev) or `FOUNDRY_PROJECT_ENDPOINT` (auto-injected in hosted containers).
|
||||
- `AZURE_AI_MODEL_DEPLOYMENT_NAME` (default `gpt-4o`).
|
||||
- `TOOLBOX_NAME` (default `my-toolset`). Use `TOOLBOX_NAME`, not `FOUNDRY_TOOLBOX_NAME`: all `FOUNDRY_*` env-var names are reserved by the Foundry platform and rejected at agent-create, so a `FOUNDRY_*`-named value would not survive deployment.
|
||||
|
||||
The `Foundry.Hosting` package builds the toolbox proxy URL from `FOUNDRY_PROJECT_ENDPOINT` as `{FOUNDRY_PROJECT_ENDPOINT}/toolboxes/{TOOLBOX_NAME}/mcp?api-version=v1` per [`tools-integration-spec.md`](https://github.com/microsoft/AgentSchema/blob/main/specs/agents/hosted_agents/container-spec/docs/tools-integration-spec.md) §2–§3.
|
||||
|
||||
## Run
|
||||
## Running directly (contributors)
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Toolbox
|
||||
dotnet run --tl:off
|
||||
```
|
||||
|
||||
### Test it
|
||||
|
||||
Using the Azure Developer CLI:
|
||||
|
||||
```powershell
|
||||
azd ai agent invoke --local "What tools do you have available, and what can they do?"
|
||||
```
|
||||
|
||||
## Running with Docker
|
||||
|
||||
### 1. Publish for the container runtime
|
||||
|
||||
```powershell
|
||||
dotnet publish -c Debug -f net10.0 -r linux-musl-x64 --self-contained false -o out
|
||||
```
|
||||
|
||||
### 2. Build and run
|
||||
|
||||
```powershell
|
||||
docker build -f Dockerfile.contributor -t hosted-toolbox .
|
||||
|
||||
$env:AZURE_BEARER_TOKEN = (az account get-access-token --resource https://ai.azure.com --query accessToken -o tsv)
|
||||
|
||||
docker run --rm -p 8088:8088 `
|
||||
-e AGENT_NAME=hosted-toolbox-agent `
|
||||
-e AZURE_BEARER_TOKEN=$env:AZURE_BEARER_TOKEN `
|
||||
--env-file .env `
|
||||
hosted-toolbox
|
||||
```
|
||||
|
||||
## Deploying to Foundry (azd spec)
|
||||
|
||||
This sample includes an `azd` manifest (`agent.manifest.yaml`) and hosted agent spec (`agent.yaml`) for deployment to Foundry.
|
||||
|
||||
Initialize an `azd` project from this sample's manifest:
|
||||
|
||||
```powershell
|
||||
mkdir hosted-toolbox; cd hosted-toolbox
|
||||
azd ai agent init -m https://github.com/microsoft/agent-framework/blob/main/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Toolbox/agent.manifest.yaml
|
||||
```
|
||||
|
||||
Then deploy:
|
||||
|
||||
```powershell
|
||||
azd deploy
|
||||
```
|
||||
|
||||
If you need to override defaults, set deployment-time environment variables in the `azd` environment before deploying:
|
||||
|
||||
```powershell
|
||||
azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME gpt-4o
|
||||
azd env set TOOLBOX_NAME my-toolset
|
||||
```
|
||||
|
||||
For end-to-end hosted agent deployment guidance, see the [official deployment guide](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/deploy-hosted-agent).
|
||||
|
||||
---
|
||||
|
||||
## NuGet package users
|
||||
|
||||
Use the standard `Dockerfile` instead of `Dockerfile.contributor`. See the commented section in `HostedToolbox.csproj` for the `PackageReference` alternative.
|
||||
|
||||
## Related samples
|
||||
|
||||
- [`Hosted-Toolbox-AuthPaths/`](../Hosted-Toolbox-AuthPaths/) — extends this pattern with a three-tool toolbox demonstrating different MCP-tool authentication paths (key, Entra agent identity, inline `Authorization`), driven by the shared `Using-Samples/SimpleAgent/` REPL.
|
||||
- [`Hosted-Toolbox-AuthPaths/`](../Hosted-Toolbox-AuthPaths/) — same hosting bones as this sample, but the toolbox bundles three MCP tools each authenticated differently (key, Entra agent identity, inline `Authorization`), driven by the shared `Using-Samples/SimpleAgent/` REPL.
|
||||
- [`Hosted-McpTools/`](../Hosted-McpTools/) — contrasts client-side `McpClient` vs server-side `HostedMcpServerTool` for non-toolbox MCP servers.
|
||||
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/AgentManifest.yaml
|
||||
name: hosted-toolbox
|
||||
displayName: "Hosted Toolbox"
|
||||
|
||||
description: >
|
||||
A hosted agent that loads its tools from a single Foundry Toolbox via the
|
||||
AF Foundry hosting bridge. AddFoundryToolboxes(name) connects to the Foundry
|
||||
Toolboxes MCP proxy at startup and exposes the toolbox's bundled tools to the
|
||||
agent on every request. The toolbox itself is provisioned out of band; see this
|
||||
sample's README for the portal walkthrough.
|
||||
|
||||
metadata:
|
||||
tags:
|
||||
- AI Agent Hosting
|
||||
- Azure AI AgentServer
|
||||
- Responses Protocol
|
||||
- Agent Framework
|
||||
- Foundry Toolbox
|
||||
- MCP
|
||||
|
||||
template:
|
||||
name: hosted-toolbox
|
||||
kind: hosted
|
||||
protocols:
|
||||
- protocol: responses
|
||||
version: 1.0.0
|
||||
resources:
|
||||
cpu: "0.25"
|
||||
memory: 0.5Gi
|
||||
environment_variables:
|
||||
- name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
value: "{{AZURE_AI_MODEL_DEPLOYMENT_NAME}}"
|
||||
- name: TOOLBOX_NAME
|
||||
value: "{{TOOLBOX_NAME}}"
|
||||
parameters:
|
||||
properties:
|
||||
- name: TOOLBOX_NAME
|
||||
type: string
|
||||
default: "my-toolset"
|
||||
description: "Name of the Foundry Toolbox to load at runtime."
|
||||
resources:
|
||||
- kind: model
|
||||
id: gpt-4o
|
||||
name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
- kind: toolbox
|
||||
name: "{{TOOLBOX_NAME}}"
|
||||
tools: []
|
||||
@@ -0,0 +1,9 @@
|
||||
# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/ContainerAgent.yaml
|
||||
kind: hosted
|
||||
name: hosted-toolbox
|
||||
protocols:
|
||||
- protocol: responses
|
||||
version: 1.0.0
|
||||
resources:
|
||||
cpu: "0.25"
|
||||
memory: 0.5Gi
|
||||
+1
-1
@@ -2,5 +2,5 @@ FOUNDRY_PROJECT_ENDPOINT=<your-azure-ai-project-endpoint>
|
||||
ASPNETCORE_URLS=http://+:8088
|
||||
ASPNETCORE_ENVIRONMENT=Development
|
||||
FOUNDRY_MODEL=gpt-5
|
||||
FOUNDRY_TOOLBOX_NAME=<your-toolbox-name>
|
||||
TOOLBOX_NAME=<your-toolbox-name>
|
||||
AZURE_BEARER_TOKEN=DefaultAzureCredential
|
||||
|
||||
+12
-5
@@ -7,9 +7,16 @@
|
||||
// AgentSkillsProviderBuilder.UseMcpSkills().
|
||||
//
|
||||
// Required environment variables:
|
||||
// FOUNDRY_PROJECT_ENDPOINT - Azure AI Foundry project endpoint
|
||||
// FOUNDRY_TOOLBOX_NAME - Name of the Foundry Toolbox to connect to
|
||||
// FOUNDRY_MODEL - Model deployment name (default: gpt-5)
|
||||
// FOUNDRY_PROJECT_ENDPOINT - Foundry project endpoint
|
||||
// TOOLBOX_NAME - Name of the Foundry Toolbox to connect to
|
||||
//
|
||||
// Optional:
|
||||
// FOUNDRY_MODEL - Model deployment name (default: gpt-5)
|
||||
//
|
||||
// NOTE: All FOUNDRY_* and AGENT_* env-var prefixes (other than the platform-injected ones
|
||||
// listed above) are reserved by the Foundry container platform and rejected at agent-create.
|
||||
// Use TOOLBOX_NAME, not FOUNDRY_TOOLBOX_NAME, for the sample-owned toolbox name so it
|
||||
// survives deployment.
|
||||
|
||||
using System.Net.Http.Headers;
|
||||
using Azure.AI.Projects;
|
||||
@@ -27,8 +34,8 @@ Env.TraversePath().Load();
|
||||
var projectEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
var deployment = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5";
|
||||
var toolboxName = Environment.GetEnvironmentVariable("FOUNDRY_TOOLBOX_NAME")
|
||||
?? throw new InvalidOperationException("FOUNDRY_TOOLBOX_NAME is not set.");
|
||||
var toolboxName = Environment.GetEnvironmentVariable("TOOLBOX_NAME")
|
||||
?? throw new InvalidOperationException("TOOLBOX_NAME is not set.");
|
||||
|
||||
// Build the Toolbox MCP URL from the project endpoint and toolbox name.
|
||||
var toolboxMcpServerUrl = $"{projectEndpoint.TrimEnd('/')}/toolboxes/{toolboxName}/mcp?api-version=v1";
|
||||
|
||||
+3
-3
@@ -13,7 +13,7 @@ This way the full skill body and resources are only loaded when the agent actual
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-5`)
|
||||
- A Foundry project with a deployed model (e.g., `gpt-5`)
|
||||
- A Foundry Toolbox already configured with skills provisioned
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
@@ -25,14 +25,14 @@ Copy the template and fill in your values:
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint and toolbox name:
|
||||
Edit `.env` and set your Foundry project endpoint and toolbox name:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
ASPNETCORE_URLS=http://+:8088
|
||||
ASPNETCORE_ENVIRONMENT=Development
|
||||
FOUNDRY_MODEL=gpt-5
|
||||
FOUNDRY_TOOLBOX_NAME=my-toolbox
|
||||
TOOLBOX_NAME=my-toolbox
|
||||
```
|
||||
|
||||
> **Note:** `.env` is gitignored. The `.env.example` template is checked in as a reference.
|
||||
|
||||
+3
-3
@@ -30,11 +30,11 @@ template:
|
||||
environment_variables:
|
||||
- name: FOUNDRY_MODEL
|
||||
value: "{{FOUNDRY_MODEL}}"
|
||||
- name: FOUNDRY_TOOLBOX_NAME
|
||||
value: "{{FOUNDRY_TOOLBOX_NAME}}"
|
||||
- name: TOOLBOX_NAME
|
||||
value: "{{TOOLBOX_NAME}}"
|
||||
parameters:
|
||||
properties:
|
||||
- name: FOUNDRY_TOOLBOX_NAME
|
||||
- name: TOOLBOX_NAME
|
||||
secret: false
|
||||
description: Name of the Foundry Toolbox to connect to for MCP skill discovery
|
||||
resources:
|
||||
|
||||
+2
-2
@@ -10,5 +10,5 @@ resources:
|
||||
environment_variables:
|
||||
- name: FOUNDRY_MODEL
|
||||
value: ${FOUNDRY_MODEL}
|
||||
- name: FOUNDRY_TOOLBOX_NAME
|
||||
value: ${FOUNDRY_TOOLBOX_NAME}
|
||||
- name: TOOLBOX_NAME
|
||||
value: ${TOOLBOX_NAME}
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ A hosted agent server demonstrating two patterns in a single app:
|
||||
|
||||
Both agents are served over the Responses protocol. The server also exposes interactive web demos at `/tool-demo` and `/workflow-demo`.
|
||||
|
||||
> Unlike the other samples in this folder, this one connects to an **Azure OpenAI** resource directly (not an Azure AI Foundry project endpoint).
|
||||
> Unlike the other samples in this folder, this one connects to an **Azure OpenAI** resource directly (not a Foundry project endpoint).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ A hosted agent that demonstrates **multi-agent workflow orchestration**. Three t
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `hosted-workflow-simple`)
|
||||
- A Foundry project with a deployed model (e.g., `hosted-workflow-simple`)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
@@ -16,7 +16,7 @@ Copy the template and fill in your project endpoint:
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint:
|
||||
Edit `.env` and set your Foundry project endpoint:
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# Using-Samples — client REPLs for the hosted agents
|
||||
|
||||
This folder holds small **client** console apps that connect to the **server** samples in the
|
||||
sibling `Hosted-*` folders. Each `Hosted-*` project is an agent you host (locally with
|
||||
`dotnet run` or deployed to Foundry); the projects here are the thing that *talks* to them.
|
||||
|
||||
## Why these exist
|
||||
|
||||
A hosted Foundry agent is an HTTP server, not a chat UI. It exposes only the per-agent OpenAI
|
||||
endpoint shape that the platform routes to:
|
||||
|
||||
```
|
||||
{FOUNDRY_PROJECT_ENDPOINT}/agents/{AZURE_AI_AGENT_NAME}/endpoint/protocols/openai
|
||||
```
|
||||
|
||||
There is no built-in console to poke it with. To actually exercise an agent — send a prompt,
|
||||
watch it call its tools, read the streamed answer — you need a client that builds a
|
||||
`FoundryAgent` against that endpoint and drives a conversation. That is all these REPLs do:
|
||||
|
||||
1. Read `FOUNDRY_PROJECT_ENDPOINT` + `AZURE_AI_AGENT_NAME` from the environment.
|
||||
2. Derive the per-agent OpenAI endpoint URL.
|
||||
3. `AIProjectClient(...).AsAIAgent(agentEndpoint)` → `FoundryAgent`.
|
||||
4. Loop: read a line, `RunStreamingAsync`, print the streamed reply.
|
||||
|
||||
The client is deliberately dumb. It knows nothing about tools, files, toolboxes, or auth — all
|
||||
of that is the hosted agent's concern on the server side. Swapping which agent you chat with is
|
||||
just a matter of changing `AZURE_AI_AGENT_NAME`.
|
||||
|
||||
## Local HTTP dev
|
||||
|
||||
When the target is a local `http://localhost:8088` dev server, the REPLs install a small
|
||||
`HttpSchemeRewritePolicy`: `AIProjectClient`/`BearerTokenPolicy` require HTTPS, so the client
|
||||
presents the endpoint as `https://` to satisfy the TLS check, then rewrites the scheme back to
|
||||
`http://` right before the request hits the wire. This is local-development only.
|
||||
|
||||
## The clients
|
||||
|
||||
| Client | What it targets | Notes |
|
||||
|---|---|---|
|
||||
| [`SimpleAgent/`](./SimpleAgent/) | Any hosted agent | Generic, agent-agnostic REPL. Point it at any `Hosted-*` server via `AZURE_AI_AGENT_NAME`. Used by `Hosted-Toolbox`, `Hosted-Toolbox-AuthPaths`, and `Hosted-McpTools`. |
|
||||
| [`SessionFilesClient/`](./SessionFilesClient/) | [`Hosted-Files`](../Hosted-Files/) | Same shape as `SimpleAgent`, framed around the bundled-files demo. |
|
||||
|
||||
## Configuration (common to all clients)
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<host>/api/projects/<project>
|
||||
AZURE_AI_AGENT_NAME=<registered-server-side-agent-name>
|
||||
```
|
||||
|
||||
Both are required. Authenticate with `az login` before running. See each client's own README for
|
||||
its end-to-end walkthrough.
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
# SimpleAgent
|
||||
|
||||
A generic, agent-agnostic chat REPL for any hosted Foundry agent. Point it at a running
|
||||
`Hosted-*` agent via `AZURE_AI_AGENT_NAME`, and it builds a `FoundryAgent` against that agent's
|
||||
per-agent OpenAI endpoint and streams replies. This is the shared client that `Hosted-Toolbox`,
|
||||
`Hosted-Toolbox-AuthPaths`, and `Hosted-McpTools` reference for their end-to-end demos.
|
||||
|
||||
It knows nothing about the agent's tools, toolboxes, files, or auth — those are entirely the
|
||||
server's concern. Changing which agent you chat with is just a different `AZURE_AI_AGENT_NAME`.
|
||||
See [`../README.md`](../README.md) for why these client REPLs exist at all.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- A running hosted agent (any `Hosted-*` sample, locally via `dotnet run` or deployed to Foundry)
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
|
||||
```env
|
||||
FOUNDRY_PROJECT_ENDPOINT=https://<host>/api/projects/<project>
|
||||
AZURE_AI_AGENT_NAME=<registered-server-side-agent-name>
|
||||
```
|
||||
|
||||
Both are required. `FOUNDRY_PROJECT_ENDPOINT` is the Foundry project endpoint URL and
|
||||
`AZURE_AI_AGENT_NAME` is the registered server-side agent name. The sample builds the per-agent
|
||||
OpenAI endpoint URL (`{FOUNDRY_PROJECT_ENDPOINT}/agents/{AZURE_AI_AGENT_NAME}/endpoint/protocols/openai`)
|
||||
from these.
|
||||
|
||||
## Run
|
||||
|
||||
Against a local Hosted-Toolbox agent listening on `http://localhost:8088`:
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-Samples/SimpleAgent
|
||||
$env:FOUNDRY_PROJECT_ENDPOINT = "http://localhost:8088/api/projects/local"
|
||||
$env:AZURE_AI_AGENT_NAME = "hosted-toolbox-agent"
|
||||
dotnet run
|
||||
```
|
||||
|
||||
When the project endpoint is `http://`, the client presents it as `https://` to satisfy the
|
||||
bearer-token TLS check, then rewrites the scheme back to `http://` right before transport
|
||||
(local-development only).
|
||||
|
||||
## End-to-end demo
|
||||
|
||||
With a hosted agent running:
|
||||
|
||||
```text
|
||||
══════════════════════════════════════════════════════════
|
||||
Simple Agent Sample
|
||||
Connected to: https://localhost:8088/api/projects/local/agents/hosted-toolbox-agent/endpoint/protocols/openai
|
||||
Type a message or 'quit' to exit
|
||||
══════════════════════════════════════════════════════════
|
||||
|
||||
You> What tools do you have available, and what can they do?
|
||||
Agent> I have the following tools from the toolbox: ...
|
||||
|
||||
You> quit
|
||||
Goodbye!
|
||||
```
|
||||
|
||||
The client only sent a chat prompt; the agent resolved its toolbox tools server-side and answered.
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,141 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample evaluates a pre-existing Azure AI Foundry agent against a rubric evaluator
|
||||
// that was authored in the Foundry portal.
|
||||
//
|
||||
// Rubric evaluators are LLM-as-judge evaluators with custom scoring dimensions you define
|
||||
// for your domain. agent-framework consumes pre-existing rubric evaluators — they are
|
||||
// authored in the Foundry portal (or via the dedicated SDK / REST surface) and referenced
|
||||
// here by name and version.
|
||||
//
|
||||
// Prerequisites:
|
||||
// - An Azure AI Foundry project with a deployed model.
|
||||
// - A registered Foundry agent in that project (the rubric was created against this agent).
|
||||
// - A rubric evaluator already created in the Foundry portal.
|
||||
// - .env (or environment) populated with the FOUNDRY_* variables below.
|
||||
//
|
||||
// IMPORTANT: FOUNDRY_PROJECT_ENDPOINT must be the project-scoped URL
|
||||
// https://<resource>.services.ai.azure.com/api/projects/<project>
|
||||
// A bare Azure OpenAI endpoint silently fails eval submission with HTTP 500.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Foundry;
|
||||
using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals;
|
||||
|
||||
string projectEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
string model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL")
|
||||
?? throw new InvalidOperationException("FOUNDRY_MODEL is not set.");
|
||||
string agentName = Environment.GetEnvironmentVariable("FOUNDRY_AGENT_NAME")
|
||||
?? throw new InvalidOperationException("FOUNDRY_AGENT_NAME is not set.");
|
||||
string? agentVersion = Environment.GetEnvironmentVariable("FOUNDRY_AGENT_VERSION");
|
||||
string rubricName = Environment.GetEnvironmentVariable("FOUNDRY_RUBRIC_NAME")
|
||||
?? throw new InvalidOperationException("FOUNDRY_RUBRIC_NAME is not set.");
|
||||
string? rubricVersion = Environment.GetEnvironmentVariable("FOUNDRY_RUBRIC_VERSION");
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful
|
||||
// consideration in production. Prefer ManagedIdentityCredential (or a specific credential)
|
||||
// to avoid latency, unintended credential probing, and fallback security risks.
|
||||
AIProjectClient projectClient = new(new Uri(projectEndpoint), new DefaultAzureCredential());
|
||||
|
||||
// 1. Connect to the pre-existing Foundry agent the rubric was created against.
|
||||
FoundryAgent agent;
|
||||
if (agentVersion is null)
|
||||
{
|
||||
ProjectsAgentRecord agentRecord = await projectClient.AgentAdministrationClient.GetAgentAsync(agentName);
|
||||
agent = projectClient.AsAIAgent(agentRecord);
|
||||
}
|
||||
else
|
||||
{
|
||||
ProjectsAgentVersion versionRecord = await projectClient.AgentAdministrationClient.GetAgentVersionAsync(agentName, agentVersion);
|
||||
agent = projectClient.AsAIAgent(versionRecord);
|
||||
}
|
||||
|
||||
// 2. Reference the pre-existing rubric evaluator by name + version.
|
||||
// Always pin a version for reproducible CI runs; a versionless ref resolves to the
|
||||
// current version at run time and emits a Trace.TraceWarning on each criterion build.
|
||||
GeneratedEvaluatorRef rubric = rubricVersion is null
|
||||
? GeneratedEvaluatorRef.Latest(rubricName)
|
||||
: new GeneratedEvaluatorRef(rubricName, rubricVersion);
|
||||
|
||||
// 3. Mix the rubric with built-in evaluators in a single FoundryEvals config.
|
||||
// The implicit conversion lets you pass strings and refs interchangeably.
|
||||
FoundryEvals evals = new(
|
||||
projectClient,
|
||||
model,
|
||||
rubric,
|
||||
FoundryEvals.Relevance,
|
||||
FoundryEvals.Coherence);
|
||||
|
||||
// 4. Run two example queries against the agent and evaluate the outputs in one call.
|
||||
string[] queries =
|
||||
[
|
||||
"What's the weather like in Seattle?",
|
||||
"Should I bring an umbrella to London tomorrow?",
|
||||
];
|
||||
|
||||
Console.WriteLine(new string('=', 60));
|
||||
Console.WriteLine($"Evaluating '{agent.Name}' with rubric '{rubricName}' (version {rubricVersion ?? "latest"})");
|
||||
Console.WriteLine(new string('=', 60));
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(queries, evals);
|
||||
|
||||
Console.WriteLine($"Status: {results.Status}");
|
||||
Console.WriteLine($"Results: {results.Passed}/{results.Total} passed");
|
||||
if (results.ReportUrl is not null)
|
||||
{
|
||||
Console.WriteLine($"Portal: {results.ReportUrl}");
|
||||
}
|
||||
|
||||
Console.WriteLine(results.Passed == results.Total ? "[PASS] All passed" : $"[FAIL] {results.Failed} failed");
|
||||
|
||||
// 5. Print per-dimension breakdown for each evaluated item — this is the unique value
|
||||
// of a rubric evaluator over the built-in numeric ones.
|
||||
Console.WriteLine();
|
||||
Console.WriteLine(new string('=', 60));
|
||||
Console.WriteLine("Per-dimension scores");
|
||||
Console.WriteLine(new string('=', 60));
|
||||
|
||||
if (results.DetailedItems is { Count: > 0 })
|
||||
{
|
||||
for (int i = 0; i < results.DetailedItems.Count; i++)
|
||||
{
|
||||
EvalItemResult item = results.DetailedItems[i];
|
||||
Console.WriteLine($"Item {i + 1}{(i < queries.Length ? $" — \"{queries[i]}\"" : string.Empty)}");
|
||||
|
||||
foreach (EvalScoreResult score in item.Scores)
|
||||
{
|
||||
Console.WriteLine($" {score.Name}: {score.Score:F1}{(score.Passed is bool p ? (p ? " (pass)" : " (fail)") : string.Empty)}");
|
||||
if (score.Dimensions is { Count: > 0 } dims)
|
||||
{
|
||||
foreach (RubricScore d in dims)
|
||||
{
|
||||
string scoreStr = d.Score is int s ? s.ToString() : "n/a";
|
||||
Console.WriteLine($" - {d.Id}: {scoreStr} (weight={d.Weight}, applicable={d.Applicable})");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
}
|
||||
|
||||
// 6. CI quality gate — fail the build if a critical dimension drops below threshold.
|
||||
// Replace "general_quality" with whatever dimension id your rubric actually defines.
|
||||
Console.WriteLine(new string('=', 60));
|
||||
Console.WriteLine("Per-dimension quality gate");
|
||||
Console.WriteLine(new string('=', 60));
|
||||
|
||||
try
|
||||
{
|
||||
results.AssertDimensionScoreAtLeast("general_quality", minScore: 3.0, evaluator: rubricName, requireApplicable: true);
|
||||
Console.WriteLine($"[PASS] {results.ProviderName}: general_quality >= 3 on every item");
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
Console.WriteLine($"[FAIL] {results.ProviderName}: dimension gate tripped: {ex.Message}");
|
||||
System.Environment.ExitCode = 1;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
# Evaluation — Foundry Rubric
|
||||
|
||||
This sample evaluates a pre-existing Azure AI Foundry agent against a **rubric evaluator**
|
||||
authored in the Foundry portal. Rubric evaluators are LLM-as-judge evaluators with custom
|
||||
scoring dimensions you define for your domain; agent-framework references them by name and
|
||||
version, mixes them with built-in evaluators, and exposes per-dimension scores you can gate
|
||||
CI on.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Connecting to a pre-existing Foundry agent (`AgentAdministrationClient.GetAgentAsync`).
|
||||
- Referencing a pre-existing rubric evaluator via `GeneratedEvaluatorRef(name, version)`.
|
||||
- Mixing the rubric with built-in evaluators (`Relevance`, `Coherence`) in one
|
||||
`FoundryEvals` run.
|
||||
- Reading per-dimension breakdowns from `EvalScoreResult.Dimensions`.
|
||||
- Gating CI on a per-dimension threshold via
|
||||
`AgentEvaluationResults.AssertDimensionScoreAtLeast(...)`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later.
|
||||
- Azure CLI installed and authenticated (`az login`).
|
||||
- An Azure AI Foundry project with a deployed model.
|
||||
- A registered Foundry agent in that project (the agent the rubric was created against).
|
||||
- A rubric evaluator created in the Foundry portal. Creating rubrics through the portal
|
||||
currently requires picking a Foundry agent as the generation context, so this
|
||||
prerequisite is implied by having a rubric at all.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> `FOUNDRY_PROJECT_ENDPOINT` **must** be the project-scoped URL
|
||||
> `https://<resource>.services.ai.azure.com/api/projects/<project>`. A bare Azure OpenAI
|
||||
> endpoint silently fails eval submission with HTTP 500.
|
||||
|
||||
> [!NOTE]
|
||||
> An **Eval Definition** (a saved bundle of testing_criteria with `"object": "eval"`) is
|
||||
> not the same as a **Rubric Evaluator** (a standalone evaluator with dimensions, weights,
|
||||
> and a version). `GeneratedEvaluatorRef` points at the latter.
|
||||
|
||||
## Environment variables
|
||||
|
||||
```powershell
|
||||
$env:FOUNDRY_PROJECT_ENDPOINT="https://your-resource.services.ai.azure.com/api/projects/your-project"
|
||||
$env:FOUNDRY_MODEL="gpt-4o-mini"
|
||||
$env:FOUNDRY_AGENT_NAME="your-agent-name"
|
||||
$env:FOUNDRY_AGENT_VERSION="1" # optional; omit for latest
|
||||
$env:FOUNDRY_RUBRIC_NAME="your-rubric-name"
|
||||
$env:FOUNDRY_RUBRIC_VERSION="1" # optional; omit for latest (CI: pin this)
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/05-end-to-end/Evaluation
|
||||
dotnet run --project .\Evaluation_FoundryRubric
|
||||
```
|
||||
+14
-13
@@ -80,33 +80,35 @@ dotnet/samples/
|
||||
|
||||
## Default provider
|
||||
|
||||
All canonical samples (01-get-started) use **Azure OpenAI** via `AzureOpenAIClient`
|
||||
with `DefaultAzureCredential`:
|
||||
All canonical samples (01-get-started) use **Microsoft Foundry** via `AIProjectClient.AsAIAgent()` with `DefaultAzureCredential`:
|
||||
|
||||
```csharp
|
||||
using Azure.AI.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Chat;
|
||||
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT")
|
||||
?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
|
||||
var model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIAgent agent = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.GetChatClient(deploymentName)
|
||||
.AsAIAgent(instructions: "...", name: "...");
|
||||
AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.AsAIAgent(model: model, instructions: "...", name: "...");
|
||||
```
|
||||
|
||||
Environment variables:
|
||||
- `AZURE_OPENAI_ENDPOINT` — Your Azure OpenAI endpoint
|
||||
- `AZURE_OPENAI_DEPLOYMENT_NAME` — Model deployment name (defaults to `gpt-5.4-mini`)
|
||||
- `FOUNDRY_PROJECT_ENDPOINT` — Your Foundry project endpoint
|
||||
- `FOUNDRY_MODEL` — Model name (defaults to `gpt-5.4-mini`)
|
||||
|
||||
For authentication, run `az login` before running samples.
|
||||
|
||||
**Note:** Use `FoundryAgent` only when demonstrating Foundry-managed (prompt) agents specifically — see `02-agents/AgentsWithFoundry/`. For all other samples, use `AIProjectClient.AsAIAgent()`.
|
||||
|
||||
**Note:** For samples demonstrating other providers (Azure OpenAI, OpenAI, Anthropic, etc.), see `02-agents/AgentProviders/`.
|
||||
|
||||
|
||||
## Snippet tags for docs integration
|
||||
|
||||
Samples embed named snippet regions for future `:::code` integration:
|
||||
@@ -135,4 +137,3 @@ dotnet run
|
||||
- Azure Functions hosting uses `ConfigureDurableAgents(options => options.AddAIAgent(agent))`
|
||||
- Workflows use `WorkflowBuilder` with `Executor<TIn, TOut>` and edge connections
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
@@ -45,6 +46,7 @@ internal static class OutputConverter
|
||||
OutputItemMessageBuilder? currentMessageBuilder = null;
|
||||
TextContentBuilder? currentTextBuilder = null;
|
||||
StringBuilder? accumulatedText = null;
|
||||
List<Annotation>? accumulatedAnnotations = null;
|
||||
string? previousMessageId = null;
|
||||
bool hasTerminalEvent = false;
|
||||
var executorItemIds = new Dictionary<string, string>();
|
||||
@@ -60,7 +62,7 @@ internal static class OutputConverter
|
||||
if (update.RawRepresentation is WorkflowEvent workflowEvent && update.Contents.Count == 0)
|
||||
{
|
||||
// Close any open message builder before emitting workflow items
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -68,6 +70,7 @@ internal static class OutputConverter
|
||||
currentTextBuilder = null;
|
||||
currentMessageBuilder = null;
|
||||
accumulatedText = null;
|
||||
accumulatedAnnotations = null;
|
||||
previousMessageId = null;
|
||||
|
||||
foreach (var evt in EmitWorkflowEvent(stream, workflowEvent, executorItemIds))
|
||||
@@ -86,7 +89,7 @@ internal static class OutputConverter
|
||||
{
|
||||
if (!IsSameMessage(update.MessageId, previousMessageId) && currentMessageBuilder is not null)
|
||||
{
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -94,6 +97,7 @@ internal static class OutputConverter
|
||||
currentTextBuilder = null;
|
||||
currentMessageBuilder = null;
|
||||
accumulatedText = null;
|
||||
accumulatedAnnotations = null;
|
||||
}
|
||||
|
||||
previousMessageId = update.MessageId;
|
||||
@@ -115,6 +119,14 @@ internal static class OutputConverter
|
||||
yield return currentTextBuilder!.EmitDelta(textContent.Text);
|
||||
}
|
||||
|
||||
if (textContent.Annotations is { Count: > 0 })
|
||||
{
|
||||
foreach (var sdkAnnotation in ConvertToSdkAnnotations(textContent.Annotations))
|
||||
{
|
||||
(accumulatedAnnotations ??= []).Add(sdkAnnotation);
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -125,7 +137,7 @@ internal static class OutputConverter
|
||||
break;
|
||||
}
|
||||
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -133,6 +145,7 @@ internal static class OutputConverter
|
||||
currentTextBuilder = null;
|
||||
currentMessageBuilder = null;
|
||||
accumulatedText = null;
|
||||
accumulatedAnnotations = null;
|
||||
previousMessageId = null;
|
||||
|
||||
var arguments = functionCall.Arguments is not null
|
||||
@@ -149,7 +162,7 @@ internal static class OutputConverter
|
||||
|
||||
case TextReasoningContent reasoningContent:
|
||||
{
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -157,6 +170,7 @@ internal static class OutputConverter
|
||||
currentTextBuilder = null;
|
||||
currentMessageBuilder = null;
|
||||
accumulatedText = null;
|
||||
accumulatedAnnotations = null;
|
||||
previousMessageId = null;
|
||||
|
||||
var reasoningBuilder = stream.AddOutputItemReasoningItem();
|
||||
@@ -176,7 +190,7 @@ internal static class OutputConverter
|
||||
|
||||
case ToolApprovalRequestContent approvalRequest when approvalRequest.ToolCall is FunctionCallContent approvalFunctionCall:
|
||||
{
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -184,6 +198,7 @@ internal static class OutputConverter
|
||||
currentTextBuilder = null;
|
||||
currentMessageBuilder = null;
|
||||
accumulatedText = null;
|
||||
accumulatedAnnotations = null;
|
||||
previousMessageId = null;
|
||||
|
||||
// The Responses API only standardizes the MCP-flavored approval primitive.
|
||||
@@ -237,7 +252,7 @@ internal static class OutputConverter
|
||||
|
||||
case ErrorContent errorContent:
|
||||
{
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -245,6 +260,7 @@ internal static class OutputConverter
|
||||
currentTextBuilder = null;
|
||||
currentMessageBuilder = null;
|
||||
accumulatedText = null;
|
||||
accumulatedAnnotations = null;
|
||||
previousMessageId = null;
|
||||
hasTerminalEvent = true;
|
||||
|
||||
@@ -269,7 +285,7 @@ internal static class OutputConverter
|
||||
break;
|
||||
}
|
||||
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -277,6 +293,7 @@ internal static class OutputConverter
|
||||
currentTextBuilder = null;
|
||||
currentMessageBuilder = null;
|
||||
accumulatedText = null;
|
||||
accumulatedAnnotations = null;
|
||||
previousMessageId = null;
|
||||
|
||||
var outputText = EncodeFunctionResultAsJsonStringPayload(functionResult.Result);
|
||||
@@ -304,7 +321,7 @@ internal static class OutputConverter
|
||||
}
|
||||
|
||||
// Close any remaining open message
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText))
|
||||
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
|
||||
{
|
||||
yield return evt;
|
||||
}
|
||||
@@ -318,7 +335,8 @@ internal static class OutputConverter
|
||||
private static IEnumerable<ResponseStreamEvent> CloseCurrentMessage(
|
||||
OutputItemMessageBuilder? messageBuilder,
|
||||
TextContentBuilder? textBuilder,
|
||||
StringBuilder? accumulatedText)
|
||||
StringBuilder? accumulatedText,
|
||||
List<Annotation>? annotations = null)
|
||||
{
|
||||
if (messageBuilder is null)
|
||||
{
|
||||
@@ -329,6 +347,16 @@ internal static class OutputConverter
|
||||
{
|
||||
var finalText = accumulatedText?.ToString() ?? string.Empty;
|
||||
yield return textBuilder.EmitTextDone(finalText);
|
||||
|
||||
// Annotations must be emitted after EmitTextDone and before EmitDone.
|
||||
if (annotations is not null)
|
||||
{
|
||||
foreach (var annotation in annotations)
|
||||
{
|
||||
yield return textBuilder.EmitAnnotationAdded(annotation);
|
||||
}
|
||||
}
|
||||
|
||||
yield return textBuilder.EmitDone();
|
||||
}
|
||||
|
||||
@@ -338,6 +366,41 @@ internal static class OutputConverter
|
||||
private static bool IsSameMessage(string? currentId, string? previousId) =>
|
||||
currentId is not { Length: > 0 } || previousId is not { Length: > 0 } || currentId == previousId;
|
||||
|
||||
/// <summary>
|
||||
/// Converts MEAI <see cref="AIAnnotation"/> instances to Responses SDK <see cref="Annotation"/> objects.
|
||||
/// Only <see cref="CitationAnnotation"/> with a URL and at least one <see cref="TextSpanAnnotatedRegion"/>
|
||||
/// with explicit start/end indices is converted; all other shapes are skipped.
|
||||
/// </summary>
|
||||
private static IEnumerable<Annotation> ConvertToSdkAnnotations(IList<AIAnnotation> annotations)
|
||||
{
|
||||
foreach (var ann in annotations)
|
||||
{
|
||||
if (ann is not CitationAnnotation citation || citation.Url is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var regions = citation.AnnotatedRegions?
|
||||
.OfType<TextSpanAnnotatedRegion>()
|
||||
.Where(r => r.StartIndex is not null && r.EndIndex is not null)
|
||||
.ToList();
|
||||
|
||||
if (regions is not { Count: > 0 })
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var region in regions)
|
||||
{
|
||||
yield return new UrlCitationBody(
|
||||
citation.Url,
|
||||
region.StartIndex!.Value,
|
||||
region.EndIndex!.Value,
|
||||
citation.Title ?? string.Empty);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static ResponseUsage ConvertUsage(UsageDetails details, ResponseUsage? existing)
|
||||
{
|
||||
var inputTokens = details.InputTokenCount ?? 0;
|
||||
|
||||
@@ -148,19 +148,68 @@ internal static class FoundryEvalConverter
|
||||
/// <summary>
|
||||
/// Builds the <c>testing_criteria</c> array for <c>evals.create()</c>.
|
||||
/// </summary>
|
||||
/// <param name="evaluators">Evaluator names (short or fully-qualified).</param>
|
||||
/// <param name="evaluators">
|
||||
/// Evaluator specs — built-in evaluator names (short or fully-qualified) and/or
|
||||
/// <see cref="GeneratedEvaluatorRef"/> instances for pre-existing rubric evaluators.
|
||||
/// </param>
|
||||
/// <param name="model">Model deployment name for the LLM judge.</param>
|
||||
/// <param name="includeDataMapping">
|
||||
/// Whether to include field-level data mapping (required for JSONL data source).
|
||||
/// </param>
|
||||
/// <param name="includeToolDefinitions">
|
||||
/// Whether the mapped data items include tool definitions. Used to add a
|
||||
/// <c>tool_definitions</c> mapping entry for rubric evaluators (built-in evaluators
|
||||
/// derive this from their own <see cref="ToolEvaluators"/> membership).
|
||||
/// </param>
|
||||
internal static List<WireTestingCriterion> BuildTestingCriteria(
|
||||
IEnumerable<string> evaluators,
|
||||
IEnumerable<FoundryEvaluatorSpec> evaluators,
|
||||
string model,
|
||||
bool includeDataMapping = false)
|
||||
bool includeDataMapping = false,
|
||||
bool includeToolDefinitions = false)
|
||||
{
|
||||
var criteria = new List<WireTestingCriterion>();
|
||||
foreach (var name in evaluators)
|
||||
foreach (var spec in evaluators)
|
||||
{
|
||||
if (spec.IsRubric)
|
||||
{
|
||||
var @ref = spec.GeneratedRef!;
|
||||
Dictionary<string, string>? refMapping = null;
|
||||
if (includeDataMapping)
|
||||
{
|
||||
// Rubric evaluators accept conversation arrays like agent evaluators,
|
||||
// plus tool_definitions when items are tool-aware.
|
||||
refMapping = new Dictionary<string, string>
|
||||
{
|
||||
["query"] = "{{item.query_messages}}",
|
||||
["response"] = "{{item.response_messages}}",
|
||||
};
|
||||
|
||||
if (includeToolDefinitions)
|
||||
{
|
||||
refMapping["tool_definitions"] = "{{item.tool_definitions}}";
|
||||
}
|
||||
}
|
||||
|
||||
criteria.Add(new WireTestingCriterion
|
||||
{
|
||||
Name = @ref.DisplayName ?? @ref.Name,
|
||||
EvaluatorName = @ref.Name,
|
||||
EvaluatorVersion = @ref.Version,
|
||||
InitializationParameters = new WireInitParams { DeploymentName = model },
|
||||
DataMapping = refMapping,
|
||||
});
|
||||
|
||||
if (@ref.Version is null)
|
||||
{
|
||||
System.Diagnostics.Trace.TraceWarning(
|
||||
"GeneratedEvaluatorRef '{0}' has no pinned version; the eval run will resolve to whichever version is current at execution time. Pin the version for reproducible runs.",
|
||||
@ref.Name);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var name = spec.BuiltinName!;
|
||||
var qualified = ResolveEvaluator(name);
|
||||
var shortName = name.StartsWith("builtin.", StringComparison.Ordinal)
|
||||
? name.Substring("builtin.".Length)
|
||||
@@ -248,8 +297,12 @@ internal static class FoundryEvalConverter
|
||||
/// Returns the subset of <paramref name="evaluators"/> that require a ground-truth
|
||||
/// (reference) value but cannot be evaluated because no item provided one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Rubric references (<see cref="GeneratedEvaluatorRef"/>) are skipped — they are not
|
||||
/// ground-truth–dependent on the wire.
|
||||
/// </remarks>
|
||||
internal static List<string> FindMissingGroundTruthEvaluators(
|
||||
IEnumerable<string> evaluators,
|
||||
IEnumerable<FoundryEvaluatorSpec> evaluators,
|
||||
bool hasGroundTruth)
|
||||
{
|
||||
if (hasGroundTruth)
|
||||
@@ -258,8 +311,14 @@ internal static class FoundryEvalConverter
|
||||
}
|
||||
|
||||
var missing = new List<string>();
|
||||
foreach (var name in evaluators)
|
||||
foreach (var spec in evaluators)
|
||||
{
|
||||
if (spec.IsRubric)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var name = spec.BuiltinName!;
|
||||
if (GroundTruthEvaluators.Contains(ResolveEvaluator(name)))
|
||||
{
|
||||
missing.Add(name);
|
||||
|
||||
@@ -137,6 +137,9 @@ internal sealed class WireTestingCriterion
|
||||
[JsonPropertyName("evaluator_name")]
|
||||
public required string EvaluatorName { get; init; }
|
||||
|
||||
[JsonPropertyName("evaluator_version")]
|
||||
public string? EvaluatorVersion { get; init; }
|
||||
|
||||
[JsonPropertyName("initialization_parameters")]
|
||||
public required WireInitParams InitializationParameters { get; init; }
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
|
||||
private readonly EvaluationClient _evaluationClient;
|
||||
private readonly string _model;
|
||||
private readonly string[] _evaluatorNames;
|
||||
private readonly FoundryEvaluatorSpec[] _evaluators;
|
||||
private readonly IConversationSplitter? _splitter;
|
||||
private readonly double _pollIntervalSeconds = 5.0;
|
||||
private readonly double _timeoutSeconds = 300.0;
|
||||
@@ -58,17 +58,21 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="evaluators">
|
||||
/// Names of evaluators to use (e.g., <see cref="Relevance"/>, <see cref="Coherence"/>).
|
||||
/// When empty, defaults to relevance and coherence.
|
||||
/// Evaluator specs to use. Each entry can be a built-in evaluator name (string, for example
|
||||
/// <see cref="Relevance"/>) or a <see cref="GeneratedEvaluatorRef"/> for a rubric evaluator
|
||||
/// already registered in the Foundry project. When empty, defaults to relevance, coherence,
|
||||
/// and task adherence.
|
||||
/// </param>
|
||||
public FoundryEvals(AIProjectClient projectClient, string model, params string[] evaluators)
|
||||
public FoundryEvals(AIProjectClient projectClient, string model, params FoundryEvaluatorSpec[] evaluators)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(projectClient);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(model);
|
||||
ArgumentNullException.ThrowIfNull(evaluators);
|
||||
EnsureAllSpecsValid(evaluators, nameof(evaluators));
|
||||
|
||||
this._evaluationClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient();
|
||||
this._model = model;
|
||||
this._evaluatorNames = evaluators.Length > 0
|
||||
this._evaluators = evaluators.Length > 0
|
||||
? evaluators
|
||||
: [Relevance, Coherence, TaskAdherence];
|
||||
}
|
||||
@@ -84,14 +88,14 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
/// or a custom <see cref="IConversationSplitter"/> implementation.
|
||||
/// </param>
|
||||
/// <param name="evaluators">
|
||||
/// Names of evaluators to use (e.g., <see cref="Relevance"/>, <see cref="Coherence"/>).
|
||||
/// When empty, defaults to relevance and coherence.
|
||||
/// Evaluator specs (built-in names and/or <see cref="GeneratedEvaluatorRef"/> instances).
|
||||
/// When empty, defaults to relevance, coherence, and task adherence.
|
||||
/// </param>
|
||||
public FoundryEvals(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IConversationSplitter? splitter,
|
||||
params string[] evaluators)
|
||||
params FoundryEvaluatorSpec[] evaluators)
|
||||
: this(projectClient, model, evaluators)
|
||||
{
|
||||
this._splitter = splitter;
|
||||
@@ -107,14 +111,16 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
/// </param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls (default 5).</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion (default 300).</param>
|
||||
/// <param name="evaluators">Evaluator names to use.</param>
|
||||
/// <param name="evaluators">
|
||||
/// Evaluator specs (built-in names and/or <see cref="GeneratedEvaluatorRef"/> instances).
|
||||
/// </param>
|
||||
public FoundryEvals(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IConversationSplitter? splitter,
|
||||
double pollIntervalSeconds,
|
||||
double timeoutSeconds,
|
||||
params string[] evaluators)
|
||||
params FoundryEvaluatorSpec[] evaluators)
|
||||
: this(projectClient, model, splitter, evaluators)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(pollIntervalSeconds, 0);
|
||||
@@ -123,6 +129,81 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
this._timeoutSeconds = timeoutSeconds;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// string[] constructor overloads (source-compat with older API that took
|
||||
// `params string[] evaluators` before FoundryEvaluatorSpec was introduced).
|
||||
// `params` is intentionally omitted to avoid overload ambiguity with the
|
||||
// spec-based ctors at zero-args; individual string literals still resolve
|
||||
// through `params FoundryEvaluatorSpec[]` via implicit conversion.
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FoundryEvals"/> class using built-in evaluator
|
||||
/// names. Preserves source compatibility for callers that pass a <see cref="string"/> array.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="evaluators">Built-in evaluator names (for example <see cref="Relevance"/>).</param>
|
||||
public FoundryEvals(AIProjectClient projectClient, string model, string[] evaluators)
|
||||
: this(projectClient, model, ToSpecs(evaluators))
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FoundryEvals"/> class with a splitter and
|
||||
/// built-in evaluator names.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="splitter">Default conversation splitter for multi-turn conversations.</param>
|
||||
/// <param name="evaluators">Built-in evaluator names.</param>
|
||||
public FoundryEvals(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IConversationSplitter? splitter,
|
||||
string[] evaluators)
|
||||
: this(projectClient, model, splitter, ToSpecs(evaluators))
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FoundryEvals"/> class with full configuration
|
||||
/// and built-in evaluator names.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="splitter">Default conversation splitter for multi-turn conversations.</param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls.</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion.</param>
|
||||
/// <param name="evaluators">Built-in evaluator names.</param>
|
||||
public FoundryEvals(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IConversationSplitter? splitter,
|
||||
double pollIntervalSeconds,
|
||||
double timeoutSeconds,
|
||||
string[] evaluators)
|
||||
: this(projectClient, model, splitter, pollIntervalSeconds, timeoutSeconds, ToSpecs(evaluators))
|
||||
{
|
||||
}
|
||||
|
||||
private static FoundryEvaluatorSpec[] ToSpecs(string[]? evaluators)
|
||||
{
|
||||
if (evaluators is null || evaluators.Length == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var specs = new FoundryEvaluatorSpec[evaluators.Length];
|
||||
for (int i = 0; i < evaluators.Length; i++)
|
||||
{
|
||||
specs[i] = evaluators[i]
|
||||
?? throw new ArgumentException($"Evaluator name at index {i} is null.", nameof(evaluators));
|
||||
}
|
||||
|
||||
return specs;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IAgentEvaluator
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -149,10 +230,10 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
bool allHaveGroundTruth = payloads.Count > 0 && payloads.All(p => p.GroundTruth is not null);
|
||||
|
||||
// Filter out tool evaluators if no items have tools; auto-add ToolCallAccuracy if tools present
|
||||
var evaluators = FilterToolEvaluators(this._evaluatorNames, hasTools);
|
||||
if (hasTools && !evaluators.Any(e => FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e))))
|
||||
var evaluators = FilterToolEvaluators(this._evaluators, hasTools);
|
||||
if (hasTools && !HasToolEvaluator(evaluators))
|
||||
{
|
||||
evaluators = [.. evaluators, ToolCallAccuracy];
|
||||
evaluators = [.. evaluators, (FoundryEvaluatorSpec)ToolCallAccuracy];
|
||||
}
|
||||
|
||||
// Fail fast if a ground-truth evaluator (e.g. similarity) is requested but not
|
||||
@@ -178,7 +259,7 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
ItemSchema = FoundryEvalConverter.BuildItemSchema(hasContext, hasTools, hasGroundTruth),
|
||||
},
|
||||
TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
evaluators, this._model, includeDataMapping: true),
|
||||
evaluators, this._model, includeDataMapping: true, includeToolDefinitions: hasTools),
|
||||
};
|
||||
|
||||
var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions);
|
||||
@@ -270,6 +351,47 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
// Static evaluation methods (traces and targets)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Source-compat overload of <see cref="EvaluateTracesAsync(AIProjectClient, string, IEnumerable{string}, IEnumerable{string}, string, int, FoundryEvaluatorSpec[], string, double, double, CancellationToken)"/>
|
||||
/// that accepts a <see cref="string"/> array of built-in evaluator names.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="responseIds">Evaluate specific Responses API response IDs.</param>
|
||||
/// <param name="traceIds">Evaluate specific OTel trace IDs from App Insights.</param>
|
||||
/// <param name="agentId">Filter traces by agent ID (used with <paramref name="lookbackHours"/>).</param>
|
||||
/// <param name="lookbackHours">Hours of trace history to evaluate.</param>
|
||||
/// <param name="evaluators">Built-in evaluator names. Each is wrapped via <see cref="FoundryEvaluatorSpec(string)"/>.</param>
|
||||
/// <param name="evalName">Display name for the evaluation.</param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls.</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results with status, report URL, and per-item details.</returns>
|
||||
public static Task<AgentEvaluationResults> EvaluateTracesAsync(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IEnumerable<string>? responseIds,
|
||||
IEnumerable<string>? traceIds,
|
||||
string? agentId,
|
||||
int lookbackHours,
|
||||
string[]? evaluators = null,
|
||||
string evalName = "Agent Framework Trace Eval",
|
||||
double pollIntervalSeconds = 5.0,
|
||||
double timeoutSeconds = 300.0,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> EvaluateTracesAsync(
|
||||
projectClient,
|
||||
model,
|
||||
responseIds,
|
||||
traceIds,
|
||||
agentId,
|
||||
lookbackHours,
|
||||
ToSpecs(evaluators) is { Length: > 0 } specs ? specs : null,
|
||||
evalName,
|
||||
pollIntervalSeconds,
|
||||
timeoutSeconds,
|
||||
cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates agent behavior from Responses API response IDs, OTel traces, or agent activity.
|
||||
/// </summary>
|
||||
@@ -287,7 +409,11 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
/// <param name="traceIds">Evaluate specific OTel trace IDs from App Insights.</param>
|
||||
/// <param name="agentId">Filter traces by agent ID (used with <paramref name="lookbackHours"/>).</param>
|
||||
/// <param name="lookbackHours">Hours of trace history to evaluate (default 24).</param>
|
||||
/// <param name="evaluators">Evaluator names. Defaults to relevance, coherence, and task adherence.</param>
|
||||
/// <param name="evaluators">
|
||||
/// Evaluator specs. Each entry can be a built-in evaluator name (string) or a
|
||||
/// <see cref="GeneratedEvaluatorRef"/> for a rubric evaluator. Defaults to relevance,
|
||||
/// coherence, and task adherence.
|
||||
/// </param>
|
||||
/// <param name="evalName">Display name for the evaluation.</param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls (default 5).</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion (default 300).</param>
|
||||
@@ -300,7 +426,7 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
IEnumerable<string>? traceIds = null,
|
||||
string? agentId = null,
|
||||
int lookbackHours = 24,
|
||||
string[]? evaluators = null,
|
||||
FoundryEvaluatorSpec[]? evaluators = null,
|
||||
string evalName = "Agent Framework Trace Eval",
|
||||
double pollIntervalSeconds = 5.0,
|
||||
double timeoutSeconds = 300.0,
|
||||
@@ -320,9 +446,10 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
}
|
||||
|
||||
var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient();
|
||||
var resolvedEvaluators = evaluators is { Length: > 0 }
|
||||
FoundryEvaluatorSpec[] resolvedEvaluators = evaluators is { Length: > 0 }
|
||||
? evaluators
|
||||
: [Relevance, Coherence, TaskAdherence];
|
||||
EnsureAllSpecsValid(resolvedEvaluators, nameof(evaluators));
|
||||
|
||||
// Create the evaluation definition with the appropriate data source scenario
|
||||
object dataSourceConfig;
|
||||
@@ -429,6 +556,41 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Source-compat overload of <see cref="EvaluateFoundryTargetAsync(AIProjectClient, string, IDictionary{string, object}, IEnumerable{string}, FoundryEvaluatorSpec[], string, double, double, CancellationToken)"/>
|
||||
/// that accepts a <see cref="string"/> array of built-in evaluator names.
|
||||
/// </summary>
|
||||
/// <param name="projectClient">The Azure AI Foundry project client.</param>
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="target">Target configuration (must include a "type" key).</param>
|
||||
/// <param name="testQueries">Queries for Foundry to send to the target.</param>
|
||||
/// <param name="evaluators">Built-in evaluator names. Each is wrapped via <see cref="FoundryEvaluatorSpec(string)"/>.</param>
|
||||
/// <param name="evalName">Display name for the evaluation.</param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls.</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results with status, report URL, and per-item details.</returns>
|
||||
public static Task<AgentEvaluationResults> EvaluateFoundryTargetAsync(
|
||||
AIProjectClient projectClient,
|
||||
string model,
|
||||
IDictionary<string, object> target,
|
||||
IEnumerable<string> testQueries,
|
||||
string[]? evaluators = null,
|
||||
string evalName = "Agent Framework Target Eval",
|
||||
double pollIntervalSeconds = 5.0,
|
||||
double timeoutSeconds = 300.0,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> EvaluateFoundryTargetAsync(
|
||||
projectClient,
|
||||
model,
|
||||
target,
|
||||
testQueries,
|
||||
ToSpecs(evaluators) is { Length: > 0 } specs ? specs : null,
|
||||
evalName,
|
||||
pollIntervalSeconds,
|
||||
timeoutSeconds,
|
||||
cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates a Foundry-registered agent or model deployment.
|
||||
/// </summary>
|
||||
@@ -440,7 +602,10 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
/// <param name="model">Model deployment name for the LLM judge evaluator.</param>
|
||||
/// <param name="target">Target configuration (must include a "type" key, e.g. "azure_ai_agent").</param>
|
||||
/// <param name="testQueries">Queries for Foundry to send to the target.</param>
|
||||
/// <param name="evaluators">Evaluator names. Defaults to relevance, coherence, and task adherence.</param>
|
||||
/// <param name="evaluators">
|
||||
/// Evaluator specs (built-in names and/or <see cref="GeneratedEvaluatorRef"/> instances).
|
||||
/// Defaults to relevance, coherence, and task adherence.
|
||||
/// </param>
|
||||
/// <param name="evalName">Display name for the evaluation.</param>
|
||||
/// <param name="pollIntervalSeconds">Seconds between status polls (default 5).</param>
|
||||
/// <param name="timeoutSeconds">Maximum seconds to wait for completion (default 300).</param>
|
||||
@@ -451,7 +616,7 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
string model,
|
||||
IDictionary<string, object> target,
|
||||
IEnumerable<string> testQueries,
|
||||
string[]? evaluators = null,
|
||||
FoundryEvaluatorSpec[]? evaluators = null,
|
||||
string evalName = "Agent Framework Target Eval",
|
||||
double pollIntervalSeconds = 5.0,
|
||||
double timeoutSeconds = 300.0,
|
||||
@@ -473,9 +638,10 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
}
|
||||
|
||||
var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient();
|
||||
var resolvedEvaluators = evaluators is { Length: > 0 }
|
||||
FoundryEvaluatorSpec[] resolvedEvaluators = evaluators is { Length: > 0 }
|
||||
? evaluators
|
||||
: [Relevance, Coherence, TaskAdherence];
|
||||
EnsureAllSpecsValid(resolvedEvaluators, nameof(evaluators));
|
||||
|
||||
var createEvalPayload = new WireCreateEvalRequest
|
||||
{
|
||||
@@ -831,7 +997,13 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
passed = pp.ValueKind == JsonValueKind.True;
|
||||
}
|
||||
|
||||
scores.Add(new EvalScoreResult(name, score, passed));
|
||||
IReadOnlyList<RubricScore>? dimensions = null;
|
||||
if (r.TryGetProperty("sample", out var perResultSample))
|
||||
{
|
||||
dimensions = ParseRubricScores(perResultSample);
|
||||
}
|
||||
|
||||
scores.Add(new EvalScoreResult(name, score, passed) { Dimensions = dimensions });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -917,20 +1089,202 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
return result;
|
||||
}
|
||||
|
||||
internal static string[] FilterToolEvaluators(string[] evaluators, bool hasTools)
|
||||
private static readonly string[] s_rubricDimensionKeys = ["dimension_scores", "rubric_scores"];
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the per-dimension <see cref="RubricScore"/> list from a result-level <c>sample</c>
|
||||
/// payload, when present. Accepts several legacy/canonical shapes for forward compatibility
|
||||
/// with provider SDK changes:
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <list type="bullet">
|
||||
/// <item>
|
||||
/// <description><c>sample.properties.dimension_scores</c> (canonical Foundry shape).</description>
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// <description><c>sample.properties.rubric_scores</c> (preview / legacy key).</description>
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// <description>Top-level <c>sample.dimension_scores</c> / <c>sample.rubric_scores</c> as a
|
||||
/// defensive fallback.</description>
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// Returns <see langword="null"/> when no rubric scores are present (the evaluator was not
|
||||
/// a rubric evaluator). Malformed entries (missing <c>id</c>, <c>weight</c>, or <c>applicable</c>)
|
||||
/// are skipped without failing the whole list.
|
||||
/// </remarks>
|
||||
internal static List<RubricScore>? ParseRubricScores(JsonElement sample)
|
||||
{
|
||||
if (sample.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Prefer sample.properties.<key> then fall back to top-level sample.<key>.
|
||||
if (sample.TryGetProperty("properties", out var properties)
|
||||
&& properties.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
foreach (var key in s_rubricDimensionKeys)
|
||||
{
|
||||
if (properties.TryGetProperty(key, out var raw))
|
||||
{
|
||||
var parsed = ParseDimensionEntries(raw);
|
||||
if (parsed.Count > 0)
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var key in s_rubricDimensionKeys)
|
||||
{
|
||||
if (sample.TryGetProperty(key, out var raw))
|
||||
{
|
||||
var parsed = ParseDimensionEntries(raw);
|
||||
if (parsed.Count > 0)
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static List<RubricScore> ParseDimensionEntries(JsonElement raw)
|
||||
{
|
||||
var parsed = new List<RubricScore>();
|
||||
if (raw.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
foreach (var entry in raw.EnumerateArray())
|
||||
{
|
||||
if (entry.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.TryGetProperty("id", out var idProp)
|
||||
|| !entry.TryGetProperty("weight", out var weightProp)
|
||||
|| !entry.TryGetProperty("applicable", out var applicableProp))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string? id = idProp.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => idProp.GetString(),
|
||||
JsonValueKind.Number => idProp.GetRawText(),
|
||||
_ => null,
|
||||
};
|
||||
if (string.IsNullOrEmpty(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (weightProp.ValueKind != JsonValueKind.Number
|
||||
|| !weightProp.TryGetInt32(out var weight))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (applicableProp.ValueKind is not (JsonValueKind.True or JsonValueKind.False))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
int? score = null;
|
||||
if (entry.TryGetProperty("score", out var scoreProp)
|
||||
&& scoreProp.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
if (scoreProp.TryGetInt32(out var intScore))
|
||||
{
|
||||
score = intScore;
|
||||
}
|
||||
else if (scoreProp.TryGetDouble(out var doubleScore))
|
||||
{
|
||||
score = (int)doubleScore;
|
||||
}
|
||||
}
|
||||
|
||||
string reason = entry.TryGetProperty("reason", out var reasonProp)
|
||||
&& reasonProp.ValueKind == JsonValueKind.String
|
||||
? reasonProp.GetString() ?? string.Empty
|
||||
: string.Empty;
|
||||
|
||||
parsed.Add(new RubricScore(
|
||||
Id: id!,
|
||||
Score: score,
|
||||
Applicable: applicableProp.ValueKind == JsonValueKind.True,
|
||||
Weight: weight,
|
||||
Reason: reason));
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
internal static FoundryEvaluatorSpec[] FilterToolEvaluators(FoundryEvaluatorSpec[] evaluators, bool hasTools)
|
||||
{
|
||||
if (hasTools)
|
||||
{
|
||||
return evaluators;
|
||||
}
|
||||
|
||||
var filtered = Array.FindAll(evaluators, e =>
|
||||
!FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e)));
|
||||
var filtered = Array.FindAll(evaluators, spec =>
|
||||
{
|
||||
if (spec.IsRubric)
|
||||
{
|
||||
// Rubric refs are tool-aware but not tool-required; preserve them.
|
||||
return true;
|
||||
}
|
||||
|
||||
return !FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(spec.BuiltinName!));
|
||||
});
|
||||
|
||||
return filtered.Length > 0
|
||||
? filtered
|
||||
: throw new ArgumentException(
|
||||
"All configured evaluators require tool definitions, but no tool calls were found in the eval items. "
|
||||
+ $"Tool evaluators: {string.Join(", ", evaluators)}. Either add tool call content to your EvalItems or remove tool-type evaluators.");
|
||||
+ $"Tool evaluators: {string.Join(", ", evaluators.Select(e => e.ToString()))}. Either add tool call content to your EvalItems or remove tool-type evaluators.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates every spec in <paramref name="evaluators"/> — defensively guards against
|
||||
/// <c>default(FoundryEvaluatorSpec)</c> values that would otherwise NRE deep in the
|
||||
/// dispatch pipeline (e.g. on <c>spec.BuiltinName!</c>).
|
||||
/// </summary>
|
||||
internal static void EnsureAllSpecsValid(FoundryEvaluatorSpec[] evaluators, string paramName)
|
||||
{
|
||||
for (int i = 0; i < evaluators.Length; i++)
|
||||
{
|
||||
if (!evaluators[i].IsValid)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Invalid {nameof(FoundryEvaluatorSpec)} at index {i}: must be constructed with either a built-in " +
|
||||
$"evaluator name or a {nameof(GeneratedEvaluatorRef)}. The default struct value is not a valid spec.",
|
||||
paramName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool HasToolEvaluator(FoundryEvaluatorSpec[] evaluators)
|
||||
{
|
||||
foreach (var spec in evaluators)
|
||||
{
|
||||
if (spec.IsRubric)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(spec.BuiltinName!)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry;
|
||||
|
||||
/// <summary>
|
||||
/// Specifies a single evaluator for a <see cref="FoundryEvals"/> run — either a built-in
|
||||
/// Foundry evaluator (referenced by short or fully-qualified name) or a pre-existing rubric
|
||||
/// evaluator (referenced by <see cref="GeneratedEvaluatorRef"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Both <see cref="string"/> and <see cref="GeneratedEvaluatorRef"/> are implicitly convertible
|
||||
/// to <see cref="FoundryEvaluatorSpec"/>, so call sites can mix the two:
|
||||
/// </para>
|
||||
/// <code>
|
||||
/// var evals = new FoundryEvals(
|
||||
/// projectClient,
|
||||
/// "gpt-4o-mini",
|
||||
/// new GeneratedEvaluatorRef("policy-rubric", Version: "3"),
|
||||
/// FoundryEvals.Relevance,
|
||||
/// FoundryEvals.Coherence);
|
||||
/// </code>
|
||||
/// </remarks>
|
||||
public readonly struct FoundryEvaluatorSpec : IEquatable<FoundryEvaluatorSpec>
|
||||
{
|
||||
private FoundryEvaluatorSpec(string? builtinName, GeneratedEvaluatorRef? generatedRef)
|
||||
{
|
||||
this.BuiltinName = builtinName;
|
||||
this.GeneratedRef = generatedRef;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new <see cref="FoundryEvaluatorSpec"/> for a built-in evaluator by name
|
||||
/// (for example <c>"relevance"</c> or <c>"builtin.relevance"</c>).
|
||||
/// </summary>
|
||||
/// <param name="builtinName">Built-in evaluator name.</param>
|
||||
public FoundryEvaluatorSpec(string builtinName)
|
||||
: this(builtinName ?? throw new ArgumentNullException(nameof(builtinName)), null)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new <see cref="FoundryEvaluatorSpec"/> for a generated rubric evaluator
|
||||
/// previously registered with the provider.
|
||||
/// </summary>
|
||||
/// <param name="generatedRef">Reference to the rubric evaluator.</param>
|
||||
public FoundryEvaluatorSpec(GeneratedEvaluatorRef generatedRef)
|
||||
: this(null, generatedRef ?? throw new ArgumentNullException(nameof(generatedRef)))
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Gets the built-in evaluator name, or <see langword="null"/> when this is a rubric reference.</summary>
|
||||
public string? BuiltinName { get; }
|
||||
|
||||
/// <summary>Gets the rubric reference, or <see langword="null"/> when this is a built-in evaluator.</summary>
|
||||
public GeneratedEvaluatorRef? GeneratedRef { get; }
|
||||
|
||||
/// <summary>Gets whether this spec references a built-in evaluator.</summary>
|
||||
public bool IsBuiltin => this.BuiltinName is not null;
|
||||
|
||||
/// <summary>Gets whether this spec references a generated rubric evaluator.</summary>
|
||||
public bool IsRubric => this.GeneratedRef is not null;
|
||||
|
||||
/// <summary>Gets whether this spec is valid (i.e. references either a built-in or a rubric).</summary>
|
||||
/// <remarks>
|
||||
/// Because <see cref="FoundryEvaluatorSpec"/> is a struct, <c>default(FoundryEvaluatorSpec)</c>
|
||||
/// is a syntactically-valid but semantically-invalid value (both <see cref="BuiltinName"/> and
|
||||
/// <see cref="GeneratedRef"/> are <see langword="null"/>). Call <see cref="EnsureValid"/> at
|
||||
/// API boundaries to fail fast instead of NRE-ing later.
|
||||
/// </remarks>
|
||||
public bool IsValid => this.BuiltinName is not null || this.GeneratedRef is not null;
|
||||
|
||||
/// <summary>Validates that this spec references either a built-in evaluator or a rubric.</summary>
|
||||
/// <param name="paramName">Parameter name used in the thrown <see cref="ArgumentException"/>.</param>
|
||||
/// <exception cref="ArgumentException">Thrown when neither <see cref="BuiltinName"/> nor <see cref="GeneratedRef"/> is set.</exception>
|
||||
public void EnsureValid(string? paramName = null)
|
||||
{
|
||||
if (!this.IsValid)
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"Invalid {nameof(FoundryEvaluatorSpec)}: must be constructed with either a built-in evaluator name " +
|
||||
$"or a {nameof(GeneratedEvaluatorRef)}. The default struct value is not a valid spec.",
|
||||
paramName);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Implicit conversion from a built-in evaluator name.</summary>
|
||||
public static implicit operator FoundryEvaluatorSpec(string builtinName) => new(builtinName);
|
||||
|
||||
/// <summary>Implicit conversion from a <see cref="GeneratedEvaluatorRef"/>.</summary>
|
||||
public static implicit operator FoundryEvaluatorSpec(GeneratedEvaluatorRef generatedRef) => new(generatedRef);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool Equals(FoundryEvaluatorSpec other)
|
||||
=> this.BuiltinName == other.BuiltinName
|
||||
&& Equals(this.GeneratedRef, other.GeneratedRef);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override bool Equals(object? obj) => obj is FoundryEvaluatorSpec other && this.Equals(other);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override int GetHashCode()
|
||||
=> HashCode.Combine(this.BuiltinName, this.GeneratedRef);
|
||||
|
||||
/// <summary>Equality operator.</summary>
|
||||
public static bool operator ==(FoundryEvaluatorSpec left, FoundryEvaluatorSpec right) => left.Equals(right);
|
||||
|
||||
/// <summary>Inequality operator.</summary>
|
||||
public static bool operator !=(FoundryEvaluatorSpec left, FoundryEvaluatorSpec right) => !left.Equals(right);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override string ToString()
|
||||
=> this.IsRubric
|
||||
? $"GeneratedEvaluatorRef({this.GeneratedRef!.Name}@{this.GeneratedRef.Version ?? "latest"})"
|
||||
: this.BuiltinName ?? "<empty>";
|
||||
}
|
||||
@@ -57,7 +57,7 @@ namespace Microsoft.Agents.AI;
|
||||
/// <para>
|
||||
/// <strong>Agent decorators (each enabled by default, individually disableable):</strong>
|
||||
/// <list type="bullet">
|
||||
/// <item><description><see cref="ToolApprovalAgent"/> — "don't ask again" tool approval rules enabling safe unattended execution. Disable with <see cref="HarnessAgentOptions.DisableToolApproval"/>.</description></item>
|
||||
/// <item><description><see cref="ToolApprovalAgent"/> — "don't ask again" tool approval rules enabling safe unattended execution. Disable with <see cref="HarnessAgentOptions.DisableToolAutoApproval"/>.</description></item>
|
||||
/// <item><description><see cref="OpenTelemetryAgent"/> — OpenTelemetry instrumentation following semantic conventions for generative AI. Disable with <see cref="HarnessAgentOptions.DisableOpenTelemetry"/>.</description></item>
|
||||
/// <item><description><see cref="LoopAgent"/> — re-invokes the agent until the configured evaluators are satisfied. Applied as the outermost decorator (so each iteration is a complete agent run) and only when <see cref="HarnessAgentOptions.LoopEvaluators"/> supplies at least one evaluator; otherwise omitted.</description></item>
|
||||
/// </list>
|
||||
@@ -155,7 +155,7 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
}
|
||||
}
|
||||
|
||||
if (options?.DisableToolApproval is not true)
|
||||
if (options?.DisableToolAutoApproval is not true)
|
||||
{
|
||||
builder.UseToolApproval(options?.ToolApprovalAgentOptions);
|
||||
}
|
||||
|
||||
@@ -183,20 +183,22 @@ public sealed class HarnessAgentOptions
|
||||
public int? MaximumIterationsPerRequest { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the <see cref="ToolApprovalAgent"/> wrapper is disabled.
|
||||
/// Gets or sets a value indicating whether the <see cref="ToolApprovalAgent"/> auto-approval middleware is disabled.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When <see langword="false"/> (the default), the agent is wrapped with tool approval middleware
|
||||
/// that supports "don't ask again" auto-approval rules.
|
||||
/// This disables the tool auto-approval functionality only, keeping the tool approval flow requiring approval (for example,
|
||||
/// <see cref="ApprovalRequiredAIFunction"/> tools). This setting controls whether the agent is wrapped with the
|
||||
/// <see cref="ToolApprovalAgent"/> middleware that supports "don't ask again" and auto-approval rules.
|
||||
/// When <see langword="false"/> (the default), the middleware is added.
|
||||
/// </remarks>
|
||||
public bool DisableToolApproval { get; set; }
|
||||
public bool DisableToolAutoApproval { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the options for the <see cref="ToolApprovalAgent"/> middleware.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When <see langword="null"/>, the <see cref="ToolApprovalAgent"/> uses default settings.
|
||||
/// This property has no effect when <see cref="DisableToolApproval"/> is <see langword="true"/>.
|
||||
/// This property has no effect when <see cref="DisableToolAutoApproval"/> is <see langword="true"/>.
|
||||
/// </remarks>
|
||||
public ToolApprovalAgentOptions? ToolApprovalAgentOptions { get; set; }
|
||||
|
||||
|
||||
@@ -20,6 +20,8 @@ internal static class BuiltInFunctions
|
||||
{
|
||||
internal const string HttpPrefix = "http-";
|
||||
internal const string McpToolPrefix = "mcptool-";
|
||||
internal const string StatusFunctionSuffix = "-status";
|
||||
internal const string RespondFunctionSuffix = "-respond";
|
||||
|
||||
private const string WaitForResponseHeaderName = "x-ms-wait-for-response";
|
||||
|
||||
@@ -90,7 +92,7 @@ internal static class BuiltInFunctions
|
||||
}
|
||||
|
||||
OrchestrationMetadata? metadata = await client.GetInstanceAsync(runId, getInputsAndOutputs: true);
|
||||
if (metadata is null)
|
||||
if (metadata is null || !IsOrchestrationOwnedByWorkflow(metadata.Name, context.FunctionDefinition.Name, StatusFunctionSuffix))
|
||||
{
|
||||
return await CreateErrorResponseAsync(req, context, HttpStatusCode.NotFound, $"Workflow run '{runId}' not found.");
|
||||
}
|
||||
@@ -146,7 +148,7 @@ internal static class BuiltInFunctions
|
||||
|
||||
// Verify the orchestration exists and is in a valid state
|
||||
OrchestrationMetadata? metadata = await client.GetInstanceAsync(runId, getInputsAndOutputs: true);
|
||||
if (metadata is null)
|
||||
if (metadata is null || !IsOrchestrationOwnedByWorkflow(metadata.Name, context.FunctionDefinition.Name, RespondFunctionSuffix))
|
||||
{
|
||||
return await CreateErrorResponseAsync(req, context, HttpStatusCode.NotFound, $"Workflow run '{runId}' not found.");
|
||||
}
|
||||
@@ -655,6 +657,39 @@ internal static class BuiltInFunctions
|
||||
return functionName[HttpPrefix.Length..];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the workflow name from the function definition name by stripping the
|
||||
/// <see cref="HttpPrefix"/> and the given suffix (e.g., "-status" or "-respond").
|
||||
/// </summary>
|
||||
internal static string GetWorkflowName(string functionName, string suffix)
|
||||
{
|
||||
if (!functionName.StartsWith(HttpPrefix, StringComparison.Ordinal) ||
|
||||
!functionName.EndsWith(suffix, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Built-in HTTP trigger function name '{functionName}' does not match the expected pattern '{HttpPrefix}<workflowName>{suffix}'.");
|
||||
}
|
||||
|
||||
return functionName[HttpPrefix.Length..^suffix.Length];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the orchestration name matches the expected orchestration for the
|
||||
/// workflow derived from the given function name and suffix.
|
||||
/// </summary>
|
||||
internal static bool IsOrchestrationOwnedByWorkflow(string orchestrationName, string functionName, string suffix)
|
||||
{
|
||||
if (!functionName.StartsWith(HttpPrefix, StringComparison.Ordinal) ||
|
||||
!functionName.EndsWith(suffix, StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string workflowName = GetWorkflowName(functionName, suffix);
|
||||
string expectedOrchestrationName = WorkflowNamingHelper.ToOrchestrationFunctionName(workflowName);
|
||||
return string.Equals(orchestrationName, expectedOrchestrationName, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a request to run an agent.
|
||||
/// </summary>
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- Scope workflow status/respond endpoints to the route workflow name ([#6608](https://github.com/microsoft/agent-framework/pull/6608))
|
||||
- Bind MCP threadId to the current agent and guard cross-agent session dispatch ([#6531](https://github.com/microsoft/agent-framework/pull/6531))
|
||||
- Support returning workflow results from HTTP trigger endpoint ([#5321](https://github.com/microsoft/agent-framework/pull/5321))
|
||||
- Added MCP tool trigger support for durable workflows ([#4768](https://github.com/microsoft/agent-framework/pull/4768))
|
||||
|
||||
+2
-2
@@ -89,7 +89,7 @@ internal sealed class DurableWorkflowsFunctionMetadataTransformer : IFunctionMet
|
||||
// Register a status endpoint if opted in via AddWorkflow(exposeStatusEndpoint: true).
|
||||
if (this._options.IsStatusEndpointEnabled(workflow.Key))
|
||||
{
|
||||
string statusFunctionName = $"{BuiltInFunctions.HttpPrefix}{workflow.Key}-status";
|
||||
string statusFunctionName = $"{BuiltInFunctions.HttpPrefix}{workflow.Key}{BuiltInFunctions.StatusFunctionSuffix}";
|
||||
if (registeredFunctions.Add(statusFunctionName))
|
||||
{
|
||||
this._logger.LogRegisteringWorkflowTrigger(workflow.Key, statusFunctionName, "http-status");
|
||||
@@ -105,7 +105,7 @@ internal sealed class DurableWorkflowsFunctionMetadataTransformer : IFunctionMet
|
||||
bool hasRequestPorts = workflow.Value.ReflectExecutors().Values.Any(b => b is RequestPortBinding);
|
||||
if (hasRequestPorts)
|
||||
{
|
||||
string respondFunctionName = $"{BuiltInFunctions.HttpPrefix}{workflow.Key}-respond";
|
||||
string respondFunctionName = $"{BuiltInFunctions.HttpPrefix}{workflow.Key}{BuiltInFunctions.RespondFunctionSuffix}";
|
||||
if (registeredFunctions.Add(respondFunctionName))
|
||||
{
|
||||
this._logger.LogRegisteringWorkflowTrigger(workflow.Key, respondFunctionName, "http-respond");
|
||||
|
||||
@@ -12,13 +12,44 @@ namespace Microsoft.Agents.AI.Workflows;
|
||||
/// </summary>
|
||||
public static partial class AgentWorkflowBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds a <see cref="Workflow"/> composed of a pipeline of agents where the output of one agent is the input to the next.
|
||||
/// </summary>
|
||||
/// <param name="chainOnlyAgentResponses">
|
||||
/// <see langword="true"/> to pass only each agent's output messages to the next agent in the sequence;
|
||||
/// <see langword="false"/> to pass the full accumulated conversation.
|
||||
/// When enabled, the workflow output also reflects only the final agent's messages,
|
||||
/// because the sequential builder stops forwarding the incoming messages to the
|
||||
/// terminal output executor.
|
||||
/// </param>
|
||||
/// <param name="agents">The sequence of agents to compose into a sequential workflow.</param>
|
||||
/// <returns>The built workflow composed of the supplied <paramref name="agents"/>, in the order in which they were yielded from the source.</returns>
|
||||
public static Workflow BuildSequential(bool chainOnlyAgentResponses, params IEnumerable<AIAgent> agents)
|
||||
=> BuildSequentialCore(workflowName: null, chainOnlyAgentResponses, agents);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a <see cref="Workflow"/> composed of a pipeline of agents where the output of one agent is the input to the next.
|
||||
/// </summary>
|
||||
/// <param name="agents">The sequence of agents to compose into a sequential workflow.</param>
|
||||
/// <returns>The built workflow composed of the supplied <paramref name="agents"/>, in the order in which they were yielded from the source.</returns>
|
||||
public static Workflow BuildSequential(params IEnumerable<AIAgent> agents)
|
||||
=> BuildSequentialCore(workflowName: null, agents);
|
||||
=> BuildSequentialCore(workflowName: null, chainOnlyAgentResponses: false, agents);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a <see cref="Workflow"/> composed of a pipeline of agents where the output of one agent is the input to the next.
|
||||
/// </summary>
|
||||
/// <param name="workflowName">The name of workflow.</param>
|
||||
/// <param name="chainOnlyAgentResponses">
|
||||
/// <see langword="true"/> to pass only each agent's output messages to the next agent in the sequence;
|
||||
/// <see langword="false"/> to pass the full accumulated conversation.
|
||||
/// When enabled, the workflow output also reflects only the final agent's messages,
|
||||
/// because the sequential builder stops forwarding the incoming messages to the
|
||||
/// terminal output executor.
|
||||
/// </param>
|
||||
/// <param name="agents">The sequence of agents to compose into a sequential workflow.</param>
|
||||
/// <returns>The built workflow composed of the supplied <paramref name="agents"/>, in the order in which they were yielded from the source.</returns>
|
||||
public static Workflow BuildSequential(string workflowName, bool chainOnlyAgentResponses, params IEnumerable<AIAgent> agents)
|
||||
=> BuildSequentialCore(workflowName, chainOnlyAgentResponses, agents);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a <see cref="Workflow"/> composed of a pipeline of agents where the output of one agent is the input to the next.
|
||||
@@ -27,13 +58,14 @@ public static partial class AgentWorkflowBuilder
|
||||
/// <param name="agents">The sequence of agents to compose into a sequential workflow.</param>
|
||||
/// <returns>The built workflow composed of the supplied <paramref name="agents"/>, in the order in which they were yielded from the source.</returns>
|
||||
public static Workflow BuildSequential(string workflowName, params IEnumerable<AIAgent> agents)
|
||||
=> BuildSequentialCore(workflowName, agents);
|
||||
=> BuildSequentialCore(workflowName, chainOnlyAgentResponses: false, agents);
|
||||
|
||||
private static Workflow BuildSequentialCore(string? workflowName, params IEnumerable<AIAgent> agents)
|
||||
private static Workflow BuildSequentialCore(string? workflowName, bool chainOnlyAgentResponses, params IEnumerable<AIAgent> agents)
|
||||
{
|
||||
Throw.IfNullOrEmpty(agents);
|
||||
|
||||
SequentialWorkflowBuilder builder = new(agents);
|
||||
SequentialWorkflowBuilder builder = new SequentialWorkflowBuilder(agents)
|
||||
.WithChainOnlyAgentResponses(chainOnlyAgentResponses);
|
||||
if (workflowName is not null)
|
||||
{
|
||||
builder.WithName(workflowName);
|
||||
|
||||
@@ -7,7 +7,6 @@ using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Agents.AI.Workflows.Checkpointing;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Execution;
|
||||
|
||||
@@ -120,13 +119,11 @@ internal sealed class EdgeMap
|
||||
return exportedStates;
|
||||
}
|
||||
|
||||
internal async ValueTask ImportStateAsync(Checkpoint checkpoint)
|
||||
internal async ValueTask ImportStateAsync(Dictionary<EdgeId, PortableValue> edgeStateData)
|
||||
{
|
||||
Dictionary<EdgeId, PortableValue> importedState = checkpoint.EdgeStateData;
|
||||
|
||||
foreach (EdgeId id in importedState.Keys)
|
||||
foreach (EdgeId id in edgeStateData.Keys)
|
||||
{
|
||||
PortableValue exportedState = importedState[id];
|
||||
PortableValue exportedState = edgeStateData[id];
|
||||
await this._statefulRunners[id].ImportStateAsync(exportedState).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,14 +97,17 @@ internal sealed class FanInEdgeRunner(IRunnerContext runContext, FanInEdgeData e
|
||||
|
||||
public ValueTask<PortableValue> ExportStateAsync()
|
||||
{
|
||||
return new(new PortableValue(this._state));
|
||||
return new(new PortableValue(this._state.Snapshot()));
|
||||
}
|
||||
|
||||
public ValueTask ImportStateAsync(PortableValue state)
|
||||
{
|
||||
if (state.Is(out FanInEdgeState? importedState))
|
||||
{
|
||||
this._state = importedState;
|
||||
// Snapshot on the way in so subsequent ProcessMessage mutations cannot alias state
|
||||
// that the checkpoint store may still be holding (e.g. an in-memory checkpoint that
|
||||
// retains the live PortableValue reference).
|
||||
this._state = importedState.Snapshot();
|
||||
return default;
|
||||
}
|
||||
|
||||
|
||||
@@ -60,4 +60,18 @@ internal sealed class FanInEdgeState
|
||||
.Select(portable => portable.ToMessageEnvelope())
|
||||
.GroupBy(messageEnvelope => messageEnvelope.Source);
|
||||
}
|
||||
|
||||
// Snapshot mutable state under _syncLock so a checkpoint export cannot observe a partially
|
||||
// updated buffer or be silently mutated afterwards (matters for in-memory checkpoint stores
|
||||
// that retain the live PortableValue reference).
|
||||
public FanInEdgeState Snapshot()
|
||||
{
|
||||
lock (this._syncLock)
|
||||
{
|
||||
return new FanInEdgeState(
|
||||
[.. this.SourceIds],
|
||||
[.. this.Unseen],
|
||||
[.. this.PendingMessages]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -418,7 +418,7 @@ internal sealed class InProcessRunner : ISuperStepRunner, ICheckpointingHandle
|
||||
|
||||
Task executorNotifyTask = this.RunContext.NotifyCheckpointLoadedAsync(cancellationToken);
|
||||
|
||||
await this.RunContext.ImportEdgeStateAsync(checkpoint).ConfigureAwait(false);
|
||||
await this.RunContext.ImportEdgeStateAsync(checkpoint.EdgeStateData).ConfigureAwait(false);
|
||||
await Task.WhenAll(executorNotifyTask,
|
||||
restoreCheckpointIndexTask.AsTask()).ConfigureAwait(false);
|
||||
|
||||
|
||||
@@ -154,6 +154,20 @@ internal sealed class InProcessRunnerContext : IRunnerContext
|
||||
|
||||
async ValueTask PrepareExternalDeliveryAsync()
|
||||
{
|
||||
if (!this._externalRequests.TryGetValue(response.RequestId, out ExternalRequest? pendingRequest))
|
||||
{
|
||||
throw new InvalidOperationException($"No pending request with ID {response.RequestId} found in the workflow context.");
|
||||
}
|
||||
|
||||
// Reject responses whose PortInfo.PortId does not match the originating request's port to
|
||||
// prevent forged routing into unrelated port-specific execution paths.
|
||||
if (!string.Equals(pendingRequest.PortInfo.PortId, response.PortInfo.PortId, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Response port id '{response.PortInfo.PortId}' does not match the originating port id for request {response.RequestId}.");
|
||||
}
|
||||
|
||||
// Consume only after validation so a rejected response leaves the legitimate one able to complete.
|
||||
if (!this.CompleteRequest(response.RequestId))
|
||||
{
|
||||
throw new InvalidOperationException($"No pending request with ID {response.RequestId} found in the workflow context.");
|
||||
@@ -418,11 +432,11 @@ internal sealed class InProcessRunnerContext : IRunnerContext
|
||||
return this._edgeMap.ExportStateAsync();
|
||||
}
|
||||
|
||||
internal ValueTask ImportEdgeStateAsync(Checkpoint checkpoint)
|
||||
internal ValueTask ImportEdgeStateAsync(Dictionary<EdgeId, PortableValue> edgeStateData)
|
||||
{
|
||||
this.CheckEnded();
|
||||
|
||||
return this._edgeMap.ImportStateAsync(checkpoint);
|
||||
return this._edgeMap.ImportStateAsync(edgeStateData);
|
||||
}
|
||||
|
||||
internal async ValueTask RepublishUnservicedRequestsAsync(CancellationToken cancellationToken = default)
|
||||
|
||||
@@ -23,6 +23,7 @@ namespace Microsoft.Agents.AI.Workflows;
|
||||
public sealed class SequentialWorkflowBuilder : OrchestrationBuilderBase<SequentialWorkflowBuilder>
|
||||
{
|
||||
private readonly List<AIAgent> _agents = [];
|
||||
private bool _chainOnlyAgentResponses;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new <see cref="SequentialWorkflowBuilder"/> with the given pipeline
|
||||
@@ -38,6 +39,24 @@ public sealed class SequentialWorkflowBuilder : OrchestrationBuilderBase<Sequent
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configures whether each downstream agent should receive only the previous agent's output,
|
||||
/// instead of the full accumulated conversation.
|
||||
/// </summary>
|
||||
/// <param name="enabled">
|
||||
/// <see langword="true"/> to pass only the previous agent's output messages to the next agent;
|
||||
/// <see langword="false"/> to pass the full conversation context.
|
||||
/// When enabled, the workflow's terminal output also contains only the final agent's
|
||||
/// messages, because incoming messages are no longer forwarded to the terminal
|
||||
/// <see cref="OutputMessagesExecutor"/>.
|
||||
/// </param>
|
||||
/// <returns>The builder instance.</returns>
|
||||
public SequentialWorkflowBuilder WithChainOnlyAgentResponses(bool enabled = true)
|
||||
{
|
||||
this._chainOnlyAgentResponses = enabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Builds the configured sequential workflow.</summary>
|
||||
public Workflow Build()
|
||||
{
|
||||
@@ -49,7 +68,7 @@ public sealed class SequentialWorkflowBuilder : OrchestrationBuilderBase<Sequent
|
||||
AIAgentHostOptions options = new()
|
||||
{
|
||||
ReassignOtherAgentsAsUsers = true,
|
||||
ForwardIncomingMessages = true,
|
||||
ForwardIncomingMessages = !this._chainOnlyAgentResponses,
|
||||
};
|
||||
|
||||
Dictionary<AIAgent, ExecutorBinding> agentMap = new(AIAgentIDEqualityComparer.Instance);
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.AI.Evaluation;
|
||||
|
||||
@@ -118,6 +119,338 @@ public sealed class AgentEvaluationResults
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asserts that every per-evaluator score on every item is at least <paramref name="minScore"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Designed for CI gates on generated rubric evaluators (for example
|
||||
/// <c>results.AssertScoreAtLeast(0.80)</c>). Walks <see cref="DetailedItems"/> across this
|
||||
/// result and any <see cref="SubResults"/> from workflow evaluations.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// When <see cref="DetailedItems"/> is <see langword="null"/>, the assertion is a no-op for
|
||||
/// this level. Providers that surface per-evaluator scores (such as Foundry) populate
|
||||
/// <see cref="DetailedItems"/>; providers that only emit aggregate <see cref="Items"/>
|
||||
/// metrics do not.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="minScore">Minimum acceptable score (inclusive).</param>
|
||||
/// <param name="evaluator">
|
||||
/// When set, only check scores whose <see cref="EvalScoreResult.Name"/> matches.
|
||||
/// </param>
|
||||
/// <param name="message">Optional custom failure message.</param>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// Thrown when any matching score is below the threshold.
|
||||
/// </exception>
|
||||
public void AssertScoreAtLeast(double minScore, string? evaluator = null, string? message = null)
|
||||
{
|
||||
var offenders = new List<string>();
|
||||
CollectScoreOffenders(this, minScore, evaluator, offenders);
|
||||
|
||||
if (offenders.Count > 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
message ?? FormatOffenders(
|
||||
$"{offenders.Count} score(s) below threshold {minScore.ToString(CultureInfo.InvariantCulture)}"
|
||||
+ (evaluator is not null ? $" for {evaluator}" : string.Empty),
|
||||
offenders));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asserts that every item's score for the given rubric dimension is at least
|
||||
/// <paramref name="minScore"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Walks <see cref="EvalScoreResult.Dimensions"/> across <see cref="DetailedItems"/>
|
||||
/// (and any <see cref="SubResults"/>) looking for the named dimension. Non-applicable
|
||||
/// dimensions are skipped by default; pass <paramref name="requireApplicable"/>=<see langword="true"/>
|
||||
/// to fail when no applicable score is produced for an item. If dimension data exists
|
||||
/// but the requested <paramref name="dimensionId"/> is never present, the assertion fails
|
||||
/// to surface likely typos or evaluator mismatches.
|
||||
/// </remarks>
|
||||
/// <param name="dimensionId">Dimension id — matches the rubric definition.</param>
|
||||
/// <param name="minScore">Minimum acceptable dimension score (inclusive).</param>
|
||||
/// <param name="evaluator">
|
||||
/// When set, only consider scores whose <see cref="EvalScoreResult.Name"/> matches.
|
||||
/// </param>
|
||||
/// <param name="requireApplicable">
|
||||
/// When <see langword="true"/>, items with no applicable score for the dimension also fail
|
||||
/// the assertion. Defaults to <see langword="false"/> (skip).
|
||||
/// </param>
|
||||
/// <param name="message">Optional custom failure message.</param>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// Thrown when the dimension fails the threshold on any item.
|
||||
/// </exception>
|
||||
public void AssertDimensionScoreAtLeast(
|
||||
string dimensionId,
|
||||
double minScore,
|
||||
string? evaluator = null,
|
||||
bool requireApplicable = false,
|
||||
string? message = null)
|
||||
{
|
||||
var offenders = new List<string>();
|
||||
var missing = new List<string>();
|
||||
CollectDimensionOffenders(this, dimensionId, minScore, evaluator, requireApplicable, offenders, missing);
|
||||
|
||||
var problems = new List<string>();
|
||||
bool hasAnyDimensionData = HasAnyDimensionData(this, evaluator);
|
||||
if (hasAnyDimensionData && !HasDimension(this, dimensionId, evaluator))
|
||||
{
|
||||
problems.Add(
|
||||
$"Dimension '{dimensionId}' was not found in results"
|
||||
+ (evaluator is not null ? $" for evaluator '{evaluator}'." : "."));
|
||||
}
|
||||
|
||||
if (offenders.Count > 0)
|
||||
{
|
||||
problems.Add(FormatOffenders(
|
||||
$"{offenders.Count} dimension score(s) for '{dimensionId}' below {minScore.ToString(CultureInfo.InvariantCulture)}",
|
||||
offenders));
|
||||
}
|
||||
|
||||
if (missing.Count > 0)
|
||||
{
|
||||
problems.Add(FormatOffenders(
|
||||
$"Dimension '{dimensionId}' not applicable on {missing.Count} item(s)",
|
||||
missing));
|
||||
}
|
||||
|
||||
if (problems.Count > 0)
|
||||
{
|
||||
throw new InvalidOperationException(message ?? string.Join("; ", problems));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asserts that no item ended in a failed or errored state. Includes any sub-results
|
||||
/// from workflow evaluations.
|
||||
/// </summary>
|
||||
/// <param name="message">Optional custom failure message.</param>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// Thrown when any item failed or errored.
|
||||
/// </exception>
|
||||
public void AssertNoFailedItems(string? message = null)
|
||||
{
|
||||
var bad = new List<string>();
|
||||
CollectFailedItems(this, bad);
|
||||
|
||||
if (bad.Count > 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
message ?? FormatOffenders($"{bad.Count} item(s) failed or errored", bad));
|
||||
}
|
||||
}
|
||||
|
||||
private static void CollectScoreOffenders(
|
||||
AgentEvaluationResults results,
|
||||
double minScore,
|
||||
string? evaluator,
|
||||
List<string> offenders)
|
||||
{
|
||||
if (results.DetailedItems is not null)
|
||||
{
|
||||
foreach (var item in results.DetailedItems)
|
||||
{
|
||||
foreach (var score in item.Scores)
|
||||
{
|
||||
if (evaluator is not null && score.Name != evaluator)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (score.Score < minScore)
|
||||
{
|
||||
offenders.Add($"{item.ItemId}/{score.Name}={score.Score.ToString("F3", CultureInfo.InvariantCulture)}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (results.SubResults is not null)
|
||||
{
|
||||
foreach (var sub in results.SubResults.Values)
|
||||
{
|
||||
CollectScoreOffenders(sub, minScore, evaluator, offenders);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void CollectDimensionOffenders(
|
||||
AgentEvaluationResults results,
|
||||
string dimensionId,
|
||||
double minScore,
|
||||
string? evaluator,
|
||||
bool requireApplicable,
|
||||
List<string> offenders,
|
||||
List<string> missing)
|
||||
{
|
||||
if (results.DetailedItems is not null)
|
||||
{
|
||||
foreach (var item in results.DetailedItems)
|
||||
{
|
||||
bool foundApplicable = false;
|
||||
foreach (var score in item.Scores)
|
||||
{
|
||||
if (evaluator is not null && score.Name != evaluator)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (score.Dimensions is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var rs in score.Dimensions)
|
||||
{
|
||||
if (rs.Id != dimensionId)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!rs.Applicable)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foundApplicable = true;
|
||||
if (rs.Score is null || rs.Score.Value < minScore)
|
||||
{
|
||||
var actual = rs.Score is null
|
||||
? "null"
|
||||
: rs.Score.Value.ToString(CultureInfo.InvariantCulture);
|
||||
offenders.Add($"{item.ItemId}/{score.Name}/{dimensionId}={actual}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (requireApplicable && !foundApplicable)
|
||||
{
|
||||
missing.Add(item.ItemId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (results.SubResults is not null)
|
||||
{
|
||||
foreach (var sub in results.SubResults.Values)
|
||||
{
|
||||
CollectDimensionOffenders(sub, dimensionId, minScore, evaluator, requireApplicable, offenders, missing);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void CollectFailedItems(AgentEvaluationResults results, List<string> bad)
|
||||
{
|
||||
if (results.DetailedItems is not null)
|
||||
{
|
||||
foreach (var item in results.DetailedItems)
|
||||
{
|
||||
if (item.IsFailed || item.IsError)
|
||||
{
|
||||
bad.Add($"{item.ItemId}:{item.Status}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (results.SubResults is not null)
|
||||
{
|
||||
foreach (var sub in results.SubResults.Values)
|
||||
{
|
||||
CollectFailedItems(sub, bad);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool HasAnyDimensionData(AgentEvaluationResults results, string? evaluator)
|
||||
{
|
||||
if (results.DetailedItems is not null)
|
||||
{
|
||||
foreach (var item in results.DetailedItems)
|
||||
{
|
||||
foreach (var score in item.Scores)
|
||||
{
|
||||
if (evaluator is not null && score.Name != evaluator)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (score.Dimensions is { Count: > 0 })
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (results.SubResults is not null)
|
||||
{
|
||||
foreach (var sub in results.SubResults.Values)
|
||||
{
|
||||
if (HasAnyDimensionData(sub, evaluator))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool HasDimension(AgentEvaluationResults results, string dimensionId, string? evaluator)
|
||||
{
|
||||
if (results.DetailedItems is not null)
|
||||
{
|
||||
foreach (var item in results.DetailedItems)
|
||||
{
|
||||
foreach (var score in item.Scores)
|
||||
{
|
||||
if (evaluator is not null && score.Name != evaluator)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (score.Dimensions is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (score.Dimensions.Any(rs => rs.Id == dimensionId))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (results.SubResults is not null)
|
||||
{
|
||||
foreach (var sub in results.SubResults.Values)
|
||||
{
|
||||
if (HasDimension(sub, dimensionId, evaluator))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string FormatOffenders(string prefix, List<string> offenders)
|
||||
{
|
||||
const int MaxShown = 5;
|
||||
if (offenders.Count <= MaxShown)
|
||||
{
|
||||
return $"{prefix}: {string.Join(", ", offenders)}";
|
||||
}
|
||||
|
||||
var shown = string.Join(", ", offenders.GetRange(0, MaxShown));
|
||||
return $"{prefix}: {shown} (+{offenders.Count - MaxShown} more)";
|
||||
}
|
||||
|
||||
private static bool ItemPassed(EvaluationResult result)
|
||||
{
|
||||
foreach (var metric in result.Metrics.Values)
|
||||
|
||||
@@ -66,7 +66,27 @@ public sealed class EvalItemResult
|
||||
/// <param name="Name">The evaluator name that produced this score.</param>
|
||||
/// <param name="Score">The numeric score value.</param>
|
||||
/// <param name="Passed">Whether the evaluator considered this a pass, or null if not determined.</param>
|
||||
public record EvalScoreResult(string Name, double Score, bool? Passed = null);
|
||||
public record EvalScoreResult(string Name, double Score, bool? Passed = null)
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the per-dimension breakdown when this evaluator is a rubric-based evaluator.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Rubric evaluators (for example, generated rubric evaluators authored in the Azure AI
|
||||
/// Foundry portal) emit one <see cref="RubricScore"/> per dimension per item alongside
|
||||
/// the overall weighted <see cref="Score"/>. Each entry preserves the dimension's
|
||||
/// applicability, weight, and the evaluator-supplied rationale.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Non-rubric evaluators (built-in quality, safety, or agent-behavior evaluators) leave
|
||||
/// this property <see langword="null"/>. Use
|
||||
/// <see cref="AgentEvaluationResults.AssertDimensionScoreAtLeast(string, double, string?, bool, string?)"/>
|
||||
/// to gate CI on a specific dimension across all items.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public IReadOnlyList<RubricScore>? Dimensions { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-evaluator pass/fail breakdown from an evaluation run.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// A reference to a generated rubric evaluator that already exists in the provider's registry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Pass instances of this class to a batch evaluator (for example
|
||||
/// <c>Microsoft.Agents.AI.Foundry.FoundryEvals</c>) to score items with a pre-existing rubric
|
||||
/// evaluator that was authored in the provider's portal or via the provider's dedicated SDK.
|
||||
/// Agent Framework is a consumer here: it does not create or modify the evaluator definition;
|
||||
/// it only references the persisted version by name.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Pinning <see cref="Version"/> is strongly recommended so evaluation runs are reproducible.
|
||||
/// A <see langword="null"/> <see cref="Version"/> resolves to whichever version is current at
|
||||
/// execution time; consuming evaluators are expected to emit a warning when a versionless
|
||||
/// reference is used. CI gates should always pass a concrete version.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="Name">
|
||||
/// Evaluator name as stored in the provider's registry (for example
|
||||
/// <c>"reservation-policy-rubric"</c>). Distinct from built-in evaluators such as
|
||||
/// <c>"relevance"</c>.
|
||||
/// </param>
|
||||
/// <param name="Version">
|
||||
/// Pinned evaluator version. <see langword="null"/> means "latest" — this is discouraged for
|
||||
/// reproducible runs and consumers may emit a warning when used.
|
||||
/// </param>
|
||||
/// <param name="DisplayName">
|
||||
/// Optional human-readable name used in result summaries. Defaults to <see cref="Name"/> when
|
||||
/// unset.
|
||||
/// </param>
|
||||
public sealed record GeneratedEvaluatorRef(
|
||||
string Name,
|
||||
string? Version = null,
|
||||
string? DisplayName = null)
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a versionless reference that resolves to the latest version of the evaluator at
|
||||
/// run time.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Discouraged for reproducible runs. Prefer the primary constructor with an explicit
|
||||
/// <see cref="Version"/> so CI and replay evaluations stay stable when the evaluator is
|
||||
/// updated in the provider's registry.
|
||||
/// </remarks>
|
||||
/// <param name="name">Evaluator name as stored in the provider's registry.</param>
|
||||
/// <param name="displayName">Optional human-readable name used in result summaries.</param>
|
||||
/// <returns>A new <see cref="GeneratedEvaluatorRef"/> with <see cref="Version"/> unset.</returns>
|
||||
public static GeneratedEvaluatorRef Latest(string name, string? displayName = null)
|
||||
=> new(name, Version: null, DisplayName: displayName);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// A single dimension's score from a rubric-based evaluator run.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Rubric evaluators (such as the generated rubric evaluators produced by Azure AI Foundry's
|
||||
/// adaptive evals) emit one <see cref="RubricScore"/> per dimension per item, alongside an
|
||||
/// overall weighted score. Attach instances to <see cref="EvalScoreResult.Dimensions"/> as
|
||||
/// a typed view of the per-dimension breakdown returned by the provider
|
||||
/// (e.g. <c>properties.dimension_scores</c>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Non-rubric evaluators (built-in quality, safety, or agent-behavior evaluators) leave
|
||||
/// <see cref="EvalScoreResult.Dimensions"/> as <see langword="null"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="Id">Dimension identifier — matches the id defined on the rubric.</param>
|
||||
/// <param name="Score">
|
||||
/// Numeric score for the dimension, or <see langword="null"/> when the dimension was marked
|
||||
/// non-applicable for this item. Foundry rubric evaluators emit integer scores on a 1–5 scale.
|
||||
/// </param>
|
||||
/// <param name="Applicable">Whether the dimension applied to this item.</param>
|
||||
/// <param name="Weight">Dimension weight, mirroring the rubric definition.</param>
|
||||
/// <param name="Reason">Short rationale produced by the evaluator.</param>
|
||||
public sealed record RubricScore(
|
||||
string Id,
|
||||
int? Score,
|
||||
bool Applicable,
|
||||
int Weight,
|
||||
string Reason);
|
||||
@@ -1,5 +1,6 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
@@ -39,10 +40,71 @@ namespace Microsoft.Agents.AI;
|
||||
/// <item><description><c>file_access_search_files</c> — Recursively search file contents using a regular expression pattern.</description></item>
|
||||
/// </list>
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// All of these tools always require approval: each is exposed as an <see cref="ApprovalRequiredAIFunction"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// To auto-approve these tools without prompting, use the <see cref="ToolApprovalAgent"/> and add one of the provided rules to
|
||||
/// <see cref="ToolApprovalAgentOptions.AutoApprovalRules"/>:
|
||||
/// <list type="bullet">
|
||||
/// <item><description>
|
||||
/// <see cref="ReadOnlyToolsAutoApprovalRule"/> — auto-approves only the read-only tools (read, list, list subdirectories,
|
||||
/// and search), while still prompting for the tools that modify the store (save and delete).
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <see cref="AllToolsAutoApprovalRule"/> — auto-approves every file access tool, including save and delete.
|
||||
/// </description></item>
|
||||
/// </list>
|
||||
/// For example, to auto-approve all file access tools:
|
||||
/// <code>
|
||||
/// builder.UseToolApproval(new ToolApprovalAgentOptions
|
||||
/// {
|
||||
/// AutoApprovalRules = [FileAccessProvider.AllToolsAutoApprovalRule],
|
||||
/// });
|
||||
/// </code>
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
|
||||
public sealed class FileAccessProvider : AIContextProvider
|
||||
{
|
||||
/// <summary>The name of the tool that saves a file.</summary>
|
||||
public const string SaveFileToolName = "file_access_save_file";
|
||||
|
||||
/// <summary>The name of the tool that reads a file.</summary>
|
||||
public const string ReadFileToolName = "file_access_read_file";
|
||||
|
||||
/// <summary>The name of the tool that deletes a file.</summary>
|
||||
public const string DeleteFileToolName = "file_access_delete_file";
|
||||
|
||||
/// <summary>The name of the tool that lists the files in a directory.</summary>
|
||||
public const string ListFilesToolName = "file_access_list_files";
|
||||
|
||||
/// <summary>The name of the tool that lists the subdirectories of a directory.</summary>
|
||||
public const string ListSubdirectoriesToolName = "file_access_list_subdirectories";
|
||||
|
||||
/// <summary>The name of the tool that searches file contents.</summary>
|
||||
public const string SearchFilesToolName = "file_access_search_files";
|
||||
|
||||
/// <summary>The names of the tools that only read from (never modify) the file store.</summary>
|
||||
private static readonly HashSet<string> s_readOnlyToolNames = new(StringComparer.Ordinal)
|
||||
{
|
||||
ReadFileToolName,
|
||||
ListFilesToolName,
|
||||
ListSubdirectoriesToolName,
|
||||
SearchFilesToolName,
|
||||
};
|
||||
|
||||
/// <summary>The names of all tools exposed by this provider.</summary>
|
||||
private static readonly HashSet<string> s_allToolNames = new(StringComparer.Ordinal)
|
||||
{
|
||||
SaveFileToolName,
|
||||
ReadFileToolName,
|
||||
DeleteFileToolName,
|
||||
ListFilesToolName,
|
||||
ListSubdirectoriesToolName,
|
||||
SearchFilesToolName,
|
||||
};
|
||||
|
||||
private const string DefaultInstructions =
|
||||
"""
|
||||
## File Access
|
||||
@@ -67,7 +129,7 @@ public sealed class FileAccessProvider : AIContextProvider
|
||||
/// The store should already be scoped to the desired folder or storage location.
|
||||
/// </param>
|
||||
/// <param name="options">Optional settings that control provider behavior. When <see langword="null"/>, defaults are used.</param>
|
||||
/// <exception cref="System.ArgumentNullException">Thrown when <paramref name="fileStore"/> is <see langword="null"/>.</exception>
|
||||
/// <exception cref="ArgumentNullException">Thrown when <paramref name="fileStore"/> is <see langword="null"/>.</exception>
|
||||
public FileAccessProvider(AgentFileStore fileStore, FileAccessProviderOptions? options = null)
|
||||
{
|
||||
Throw.IfNull(fileStore);
|
||||
@@ -76,6 +138,44 @@ public sealed class FileAccessProvider : AIContextProvider
|
||||
this._instructions = options?.Instructions ?? DefaultInstructions;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets an auto-approval rule that approves the read-only file access tools
|
||||
/// (<see cref="ReadFileToolName"/>, <see cref="ListFilesToolName"/>,
|
||||
/// <see cref="ListSubdirectoriesToolName"/>, and <see cref="SearchFilesToolName"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The tools exposed by <see cref="FileAccessProvider"/> always require approval. Add this rule to
|
||||
/// <see cref="ToolApprovalAgentOptions.AutoApprovalRules"/> to automatically approve only the tools
|
||||
/// that read from the file store, while still prompting for tools that modify it
|
||||
/// (<see cref="SaveFileToolName"/> and <see cref="DeleteFileToolName"/>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The rule matches on the tool name, returning <see langword="true"/> for read-only file access tools
|
||||
/// and <see langword="false"/> for all other tool calls so that subsequent rules continue to be evaluated.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static Func<FunctionCallContent, ValueTask<bool>> ReadOnlyToolsAutoApprovalRule { get; } =
|
||||
functionCall => new ValueTask<bool>(s_readOnlyToolNames.Contains(functionCall.Name));
|
||||
|
||||
/// <summary>
|
||||
/// Gets an auto-approval rule that approves all file access tools, including the tools that modify the
|
||||
/// file store (<see cref="SaveFileToolName"/> and <see cref="DeleteFileToolName"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The tools exposed by <see cref="FileAccessProvider"/> always require approval. Add this rule to
|
||||
/// <see cref="ToolApprovalAgentOptions.AutoApprovalRules"/> to automatically approve every file access
|
||||
/// tool without prompting the user.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The rule matches on the tool name, returning <see langword="true"/> for any file access tool
|
||||
/// and <see langword="false"/> for all other tool calls so that subsequent rules continue to be evaluated.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static Func<FunctionCallContent, ValueTask<bool>> AllToolsAutoApprovalRule { get; } =
|
||||
functionCall => new ValueTask<bool>(s_allToolNames.Contains(functionCall.Name));
|
||||
|
||||
/// <inheritdoc />
|
||||
public override IReadOnlyList<string> StateKeys => [];
|
||||
|
||||
@@ -197,14 +297,17 @@ public sealed class FileAccessProvider : AIContextProvider
|
||||
{
|
||||
var serializerOptions = AgentJsonUtilities.DefaultOptions;
|
||||
|
||||
// All file access tools always require approval. Callers can use the
|
||||
// ReadOnlyToolsAutoApprovalRule or AllToolsAutoApprovalRule with the ToolApprovalAgent
|
||||
// to automatically approve these tools.
|
||||
return
|
||||
[
|
||||
AIFunctionFactory.Create(this.SaveFileAsync, new AIFunctionFactoryOptions { Name = "file_access_save_file", SerializerOptions = serializerOptions }),
|
||||
AIFunctionFactory.Create(this.ReadFileAsync, new AIFunctionFactoryOptions { Name = "file_access_read_file", SerializerOptions = serializerOptions }),
|
||||
AIFunctionFactory.Create(this.DeleteFileAsync, new AIFunctionFactoryOptions { Name = "file_access_delete_file", SerializerOptions = serializerOptions }),
|
||||
AIFunctionFactory.Create(this.ListFilesAsync, new AIFunctionFactoryOptions { Name = "file_access_list_files", SerializerOptions = serializerOptions }),
|
||||
AIFunctionFactory.Create(this.ListSubdirectoriesAsync, new AIFunctionFactoryOptions { Name = "file_access_list_subdirectories", SerializerOptions = serializerOptions }),
|
||||
AIFunctionFactory.Create(this.SearchFilesAsync, new AIFunctionFactoryOptions { Name = "file_access_search_files", SerializerOptions = serializerOptions }),
|
||||
new ApprovalRequiredAIFunction(AIFunctionFactory.Create(this.SaveFileAsync, new AIFunctionFactoryOptions { Name = SaveFileToolName, SerializerOptions = serializerOptions })),
|
||||
new ApprovalRequiredAIFunction(AIFunctionFactory.Create(this.ReadFileAsync, new AIFunctionFactoryOptions { Name = ReadFileToolName, SerializerOptions = serializerOptions })),
|
||||
new ApprovalRequiredAIFunction(AIFunctionFactory.Create(this.DeleteFileAsync, new AIFunctionFactoryOptions { Name = DeleteFileToolName, SerializerOptions = serializerOptions })),
|
||||
new ApprovalRequiredAIFunction(AIFunctionFactory.Create(this.ListFilesAsync, new AIFunctionFactoryOptions { Name = ListFilesToolName, SerializerOptions = serializerOptions })),
|
||||
new ApprovalRequiredAIFunction(AIFunctionFactory.Create(this.ListSubdirectoriesAsync, new AIFunctionFactoryOptions { Name = ListSubdirectoriesToolName, SerializerOptions = serializerOptions })),
|
||||
new ApprovalRequiredAIFunction(AIFunctionFactory.Create(this.SearchFilesAsync, new AIFunctionFactoryOptions { Name = SearchFilesToolName, SerializerOptions = serializerOptions })),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +73,28 @@ public sealed class ToolApprovalAgent : DelegatingAIAgent
|
||||
this._jsonSerializerOptions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets an auto-approval rule that approves every tool call, regardless of tool name or arguments.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Add this rule to <see cref="ToolApprovalAgentOptions.AutoApprovalRules"/> to automatically approve all
|
||||
/// tool calls without prompting the user. This effectively disables approval prompts for every tool, so use
|
||||
/// it only when running in a fully trusted context.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// For example, to auto-approve every tool call:
|
||||
/// <code>
|
||||
/// builder.UseToolApproval(new ToolApprovalAgentOptions
|
||||
/// {
|
||||
/// AutoApprovalRules = [ToolApprovalAgent.AllToolsAutoApprovalRule],
|
||||
/// });
|
||||
/// </code>
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static Func<FunctionCallContent, ValueTask<bool>> AllToolsAutoApprovalRule { get; } =
|
||||
_ => new ValueTask<bool>(true);
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task<AgentResponse> RunCoreAsync(
|
||||
IEnumerable<ChatMessage> messages,
|
||||
@@ -741,11 +763,19 @@ public sealed class ToolApprovalAgent : DelegatingAIAgent
|
||||
/// <summary>
|
||||
/// Serializes function call arguments to a string dictionary for storage and comparison.
|
||||
/// </summary>
|
||||
private static Dictionary<string, string>? SerializeArguments(IDictionary<string, object?>? arguments, JsonSerializerOptions jsonSerializerOptions)
|
||||
/// <remarks>
|
||||
/// Always returns a non-null dictionary so that an argument-scoped standing approval
|
||||
/// (the <see cref="AlwaysApproveToolApprovalResponseContent.AlwaysApproveToolWithArguments"/>
|
||||
/// path) records an exact-arguments rule. A <see langword="null"/> or empty source dictionary
|
||||
/// yields an empty dictionary, which matches only future no-argument calls. A <see langword="null"/>
|
||||
/// value is reserved on <see cref="ToolApprovalRule.Arguments"/> for tool-level rules and is never
|
||||
/// produced here, preventing an exact-arguments approval from widening into a tool-level approval.
|
||||
/// </remarks>
|
||||
private static Dictionary<string, string> SerializeArguments(IDictionary<string, object?>? arguments, JsonSerializerOptions jsonSerializerOptions)
|
||||
{
|
||||
if (arguments is null || arguments.Count == 0)
|
||||
{
|
||||
return null;
|
||||
return new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
var serialized = new Dictionary<string, string>(arguments.Count, StringComparer.Ordinal);
|
||||
|
||||
@@ -18,9 +18,16 @@ namespace Microsoft.Agents.AI;
|
||||
/// <item><b>Tool-level</b>: When <see cref="Arguments"/> is <see langword="null"/>,
|
||||
/// all calls to the tool identified by <see cref="ToolName"/> are auto-approved.</item>
|
||||
/// <item><b>Tool+arguments</b>: When <see cref="Arguments"/> is non-null,
|
||||
/// only calls to the specified tool with exactly matching argument values are auto-approved.</item>
|
||||
/// only calls to the specified tool with exactly matching argument values are auto-approved.
|
||||
/// A non-null but <b>empty</b> dictionary matches only calls that supply no arguments; it does
|
||||
/// not widen into a tool-level rule.</item>
|
||||
/// </list>
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <see langword="null"/> is therefore reserved exclusively for tool-level approval. An
|
||||
/// argument-scoped approval (including one created from a no-argument call) is always stored
|
||||
/// as a non-null dictionary so it cannot be silently broadened to all invocations of the tool.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
|
||||
internal sealed class ToolApprovalRule
|
||||
@@ -34,7 +41,8 @@ internal sealed class ToolApprovalRule
|
||||
/// <summary>
|
||||
/// Gets or sets the specific argument values that must match for this rule to apply.
|
||||
/// When <see langword="null"/>, the rule applies to all invocations of the tool
|
||||
/// regardless of arguments.
|
||||
/// regardless of arguments. A non-null but empty dictionary applies only to
|
||||
/// invocations that supply no arguments.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Argument values are stored as their JSON-serialized string representations
|
||||
|
||||
@@ -242,10 +242,11 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
}
|
||||
|
||||
string sourceName = this._sourceName;
|
||||
static IChatClient WrapIfNeeded(IChatClient cc, string sourceName) =>
|
||||
bool enableSensitiveData = this.EnableSensitiveData;
|
||||
static IChatClient WrapIfNeeded(IChatClient cc, string sourceName, bool enableSensitiveData) =>
|
||||
cc.GetService(typeof(OpenTelemetryChatClient)) is not null
|
||||
? cc
|
||||
: cc.AsBuilder().UseOpenTelemetry(sourceName: sourceName).Build();
|
||||
: cc.AsBuilder().UseOpenTelemetry(sourceName: sourceName, configure: o => o.EnableSensitiveData = enableSensitiveData).Build();
|
||||
|
||||
if (options is ChatClientAgentRunOptions ccOptions)
|
||||
{
|
||||
@@ -253,7 +254,7 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
// If the user factory already returns an OpenTelemetry-instrumented client, don't double-wrap.
|
||||
var clone = (ChatClientAgentRunOptions)ccOptions.Clone();
|
||||
var userFactory = clone.ChatClientFactory;
|
||||
clone.ChatClientFactory = cc => WrapIfNeeded(userFactory is null ? cc : userFactory(cc), sourceName);
|
||||
clone.ChatClientFactory = cc => WrapIfNeeded(userFactory is null ? cc : userFactory(cc), sourceName, enableSensitiveData);
|
||||
return clone;
|
||||
}
|
||||
|
||||
@@ -261,7 +262,7 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
// any base AgentRunOptions properties from the caller so they reach the inner agent.
|
||||
var newOptions = new ChatClientAgentRunOptions
|
||||
{
|
||||
ChatClientFactory = cc => WrapIfNeeded(cc, sourceName),
|
||||
ChatClientFactory = cc => WrapIfNeeded(cc, sourceName, enableSensitiveData),
|
||||
};
|
||||
|
||||
if (options is not null)
|
||||
|
||||
@@ -115,13 +115,21 @@ public sealed class SessionFilesHostedAgentTests(SessionFilesHostedAgentFixture
|
||||
long expectedBytes = new FileInfo(localPath).Length;
|
||||
Assert.Equal(expectedBytes, writeResponse.BytesWritten);
|
||||
|
||||
SessionDirectoryListResponse listing = await sessionFiles.GetSessionFilesAsync(
|
||||
bool foundEntry = false;
|
||||
await foreach (SessionDirectoryEntry entry in sessionFiles.GetSessionFilesAsync(
|
||||
agentName: this._fixture.AgentName,
|
||||
sessionId: agentSessionId,
|
||||
sessionStoragePath: ".");
|
||||
Assert.Contains(
|
||||
listing.Entries,
|
||||
e => e.Name == TestDataFileName && !e.IsDirectory && e.Size == expectedBytes);
|
||||
agentSessionId: agentSessionId,
|
||||
sessionStoragePath: "."))
|
||||
{
|
||||
if (entry.Name == TestDataFileName && !entry.IsDirectory && entry.Size == expectedBytes)
|
||||
{
|
||||
foundEntry = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Assert.True(
|
||||
foundEntry,
|
||||
$"Expected session directory listing to contain '{TestDataFileName}' ({expectedBytes} bytes) as a file.");
|
||||
|
||||
// Step 4 — invoke the agent again on the SAME conversation. The platform routes back to
|
||||
// the same agent_session_id container, so the agent's ReadFile tool sees the upload.
|
||||
|
||||
@@ -1373,6 +1373,259 @@ public class OutputConverterTests
|
||||
Assert.Contains(events, e => e is ResponseFailedEvent);
|
||||
}
|
||||
|
||||
#region url_citation annotation coverage
|
||||
|
||||
/// <summary>A <see cref="MeaiTextContent"/> with a url_citation annotation emits a <see cref="ResponseOutputTextAnnotationAddedEvent"/>.</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_TextWithUrlCitationAnnotation_EmitsAnnotationEventAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var annotation = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://example.com/doc"),
|
||||
Title = "Example Document",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }]
|
||||
};
|
||||
var textContent = new MeaiTextContent("Hello") { Annotations = [annotation] };
|
||||
var update = new AgentResponseUpdate { MessageId = "msg_1", Contents = [textContent] };
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
var annotationEvent = Assert.Single(events.OfType<ResponseOutputTextAnnotationAddedEvent>());
|
||||
var urlCitation = Assert.IsType<UrlCitationBody>(annotationEvent.Annotation);
|
||||
Assert.Equal(new Uri("https://example.com/doc"), urlCitation.Url);
|
||||
Assert.Equal("Example Document", urlCitation.Title);
|
||||
Assert.Equal(0L, urlCitation.StartIndex);
|
||||
Assert.Equal(5L, urlCitation.EndIndex);
|
||||
Assert.IsType<ResponseCompletedEvent>(events[^1]);
|
||||
}
|
||||
|
||||
/// <summary>The content_part.done and output_item.done payloads carry the url_citation metadata, guarding against the empty-annotations regression where only the annotation.added event fires.</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_TextWithUrlCitationAnnotation_DoneEventsCarryAnnotationMetadataAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var annotation = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://example.com/doc"),
|
||||
Title = "Example Document",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }]
|
||||
};
|
||||
var update = new AgentResponseUpdate
|
||||
{
|
||||
MessageId = "msg_1",
|
||||
Contents = [new MeaiTextContent("Hello") { Annotations = [annotation] }]
|
||||
};
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
// content_part.done must serialize the annotation, not an empty array.
|
||||
var contentPartDone = Assert.Single(events.OfType<ResponseContentPartDoneEvent>());
|
||||
var donePart = Assert.IsType<OutputContentOutputTextContent>(contentPartDone.Part);
|
||||
var donePartCitation = Assert.IsType<UrlCitationBody>(Assert.Single(donePart.Annotations));
|
||||
Assert.Equal(new Uri("https://example.com/doc"), donePartCitation.Url);
|
||||
Assert.Equal("Example Document", donePartCitation.Title);
|
||||
|
||||
// output_item.done message content must also carry the annotation.
|
||||
var outputItemDone = Assert.Single(events.OfType<ResponseOutputItemDoneEvent>());
|
||||
var doneMessage = Assert.IsType<OutputItemMessage>(outputItemDone.Item);
|
||||
var doneText = Assert.IsType<MessageContentOutputTextContent>(Assert.Single(doneMessage.Content));
|
||||
var doneTextCitation = Assert.IsType<UrlCitationBody>(Assert.Single(doneText.Annotations));
|
||||
Assert.Equal(new Uri("https://example.com/doc"), doneTextCitation.Url);
|
||||
Assert.Equal("Example Document", doneTextCitation.Title);
|
||||
}
|
||||
|
||||
/// <summary>The annotation event must appear after the last text delta and before output_item.done.</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_TextWithAnnotation_AnnotationOrderedAfterDeltaBeforeMessageDoneAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var annotation = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://example.com/src"),
|
||||
Title = "Source",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 10, EndIndex = 20 }]
|
||||
};
|
||||
var update = new AgentResponseUpdate
|
||||
{
|
||||
MessageId = "msg_1",
|
||||
Contents = [new MeaiTextContent("text with citation") { Annotations = [annotation] }]
|
||||
};
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
var lastDeltaIdx = events.FindLastIndex(e => e is ResponseTextDeltaEvent);
|
||||
var annotationIdx = events.FindIndex(e => e is ResponseOutputTextAnnotationAddedEvent);
|
||||
var messageDoneIdx = events.FindIndex(e => e is ResponseOutputItemDoneEvent);
|
||||
|
||||
Assert.True(annotationIdx >= 0, "Expected ResponseOutputTextAnnotationAddedEvent");
|
||||
Assert.True(messageDoneIdx >= 0, "Expected ResponseOutputItemDoneEvent");
|
||||
Assert.True(lastDeltaIdx < annotationIdx, "Annotation must come after last text delta");
|
||||
Assert.True(annotationIdx < messageDoneIdx, "Annotation must come before output_item.done");
|
||||
}
|
||||
|
||||
/// <summary>Multiple annotations on one <see cref="MeaiTextContent"/> all emit events, in order.</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_TextWithMultipleAnnotations_EmitsAllAnnotationsAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var ann1 = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://a.example.com"),
|
||||
Title = "A",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 10 }]
|
||||
};
|
||||
var ann2 = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://b.example.com"),
|
||||
Title = "B",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 20, EndIndex = 30 }]
|
||||
};
|
||||
var update = new AgentResponseUpdate
|
||||
{
|
||||
MessageId = "msg_1",
|
||||
Contents = [new MeaiTextContent("text") { Annotations = [ann1, ann2] }]
|
||||
};
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
var annotationEvents = events.OfType<ResponseOutputTextAnnotationAddedEvent>().ToList();
|
||||
Assert.Equal(2, annotationEvents.Count);
|
||||
var first = Assert.IsType<UrlCitationBody>(annotationEvents[0].Annotation);
|
||||
Assert.Equal("A", first.Title);
|
||||
var second = Assert.IsType<UrlCitationBody>(annotationEvents[1].Annotation);
|
||||
Assert.Equal("B", second.Title);
|
||||
Assert.IsType<ResponseCompletedEvent>(events[^1]);
|
||||
}
|
||||
|
||||
/// <summary>Annotations on a <see cref="MeaiTextContent"/> across multiple streaming updates are all accumulated.</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_AnnotationsAcrossMultipleUpdates_AccumulatesAllAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var ann1 = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://first.example.com"),
|
||||
Title = "First",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }]
|
||||
};
|
||||
var ann2 = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://second.example.com"),
|
||||
Title = "Second",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 6, EndIndex = 11 }]
|
||||
};
|
||||
var updates = new[]
|
||||
{
|
||||
new AgentResponseUpdate { MessageId = "msg_1", Contents = [new MeaiTextContent("Hello") { Annotations = [ann1] }] },
|
||||
new AgentResponseUpdate { MessageId = "msg_1", Contents = [new MeaiTextContent(" world") { Annotations = [ann2] }] },
|
||||
};
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
var annotationEvents = events.OfType<ResponseOutputTextAnnotationAddedEvent>().ToList();
|
||||
Assert.Equal(2, annotationEvents.Count);
|
||||
Assert.IsType<ResponseCompletedEvent>(events[^1]);
|
||||
}
|
||||
|
||||
/// <summary>A <see cref="CitationAnnotation"/> without a URL is skipped (no annotation event emitted).</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_CitationAnnotationWithoutUrl_IsSkippedAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var annotation = new CitationAnnotation
|
||||
{
|
||||
Url = null,
|
||||
Title = "No URL",
|
||||
AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }]
|
||||
};
|
||||
var update = new AgentResponseUpdate
|
||||
{
|
||||
MessageId = "msg_1",
|
||||
Contents = [new MeaiTextContent("text") { Annotations = [annotation] }]
|
||||
};
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
Assert.Empty(events.OfType<ResponseOutputTextAnnotationAddedEvent>());
|
||||
Assert.IsType<ResponseCompletedEvent>(events[^1]);
|
||||
}
|
||||
|
||||
/// <summary>A <see cref="CitationAnnotation"/> without a <see cref="TextSpanAnnotatedRegion"/> is skipped.</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_CitationAnnotationWithoutRegions_IsSkippedAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var annotation = new CitationAnnotation
|
||||
{
|
||||
Url = new Uri("https://example.com"),
|
||||
Title = "No Regions",
|
||||
AnnotatedRegions = null
|
||||
};
|
||||
var update = new AgentResponseUpdate
|
||||
{
|
||||
MessageId = "msg_1",
|
||||
Contents = [new MeaiTextContent("text") { Annotations = [annotation] }]
|
||||
};
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
Assert.Empty(events.OfType<ResponseOutputTextAnnotationAddedEvent>());
|
||||
Assert.IsType<ResponseCompletedEvent>(events[^1]);
|
||||
}
|
||||
|
||||
/// <summary>A non-<see cref="CitationAnnotation"/> in Annotations is silently skipped.</summary>
|
||||
[Fact]
|
||||
public async Task ConvertUpdatesToEventsAsync_NonCitationAnnotation_IsSkippedAsync()
|
||||
{
|
||||
var (stream, _) = CreateTestStream();
|
||||
var rawAnnotation = new AIAnnotation(); // base type, not a CitationAnnotation
|
||||
var update = new AgentResponseUpdate
|
||||
{
|
||||
MessageId = "msg_1",
|
||||
Contents = [new MeaiTextContent("text") { Annotations = [rawAnnotation] }]
|
||||
};
|
||||
|
||||
var events = new List<ResponseStreamEvent>();
|
||||
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
|
||||
{
|
||||
events.Add(evt);
|
||||
}
|
||||
|
||||
Assert.Empty(events.OfType<ResponseOutputTextAnnotationAddedEvent>());
|
||||
Assert.IsType<ResponseCompletedEvent>(events[^1]);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private sealed class RawToolCallContent : ToolCallContent
|
||||
{
|
||||
public RawToolCallContent(string callId) : base(callId) { }
|
||||
|
||||
@@ -305,6 +305,71 @@ public sealed class FoundryEvalConverterTests
|
||||
Assert.Null(criteria[0].DataMapping);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_WithRubricRef_EmitsAzureAiEvaluatorWithVersion()
|
||||
{
|
||||
var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "3", DisplayName: "Policy");
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
[rubric], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var entry = criteria[0];
|
||||
Assert.Equal("azure_ai_evaluator", entry.Type);
|
||||
Assert.Equal("Policy", entry.Name);
|
||||
Assert.Equal("policy-rubric", entry.EvaluatorName);
|
||||
Assert.Equal("3", entry.EvaluatorVersion);
|
||||
Assert.Equal("gpt-4o-mini", entry.InitializationParameters.DeploymentName);
|
||||
|
||||
var mapping = entry.DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.Equal("{{item.query_messages}}", mapping["query"]);
|
||||
Assert.Equal("{{item.response_messages}}", mapping["response"]);
|
||||
Assert.False(mapping.ContainsKey("tool_definitions"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_WithVersionlessRubricRef_OmitsVersionField()
|
||||
{
|
||||
var rubric = GeneratedEvaluatorRef.Latest("policy-rubric");
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
[rubric], "gpt-4o-mini", includeDataMapping: false);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var entry = criteria[0];
|
||||
Assert.Equal("policy-rubric", entry.Name); // falls back to Name when DisplayName is null
|
||||
Assert.Equal("policy-rubric", entry.EvaluatorName);
|
||||
Assert.Null(entry.EvaluatorVersion);
|
||||
Assert.Null(entry.DataMapping);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_RubricRefWithTools_IncludesToolDefinitions()
|
||||
{
|
||||
var rubric = new GeneratedEvaluatorRef("tool-aware-rubric", Version: "1");
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
[rubric], "gpt-4o-mini", includeDataMapping: true, includeToolDefinitions: true);
|
||||
|
||||
Assert.Single(criteria);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.True(mapping.ContainsKey("tool_definitions"));
|
||||
Assert.Equal("{{item.tool_definitions}}", mapping["tool_definitions"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_MixedSpecs_PreservesOrder()
|
||||
{
|
||||
var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "2");
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["relevance", rubric, "coherence"], "gpt-4o-mini", includeDataMapping: false);
|
||||
|
||||
Assert.Equal(3, criteria.Count);
|
||||
Assert.Equal("builtin.relevance", criteria[0].EvaluatorName);
|
||||
Assert.Equal("policy-rubric", criteria[1].EvaluatorName);
|
||||
Assert.Equal("2", criteria[1].EvaluatorVersion);
|
||||
Assert.Equal("builtin.coherence", criteria[2].EvaluatorName);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.BuildItemSchema tests
|
||||
// ---------------------------------------------------------------
|
||||
@@ -391,6 +456,18 @@ public sealed class FoundryEvalConverterTests
|
||||
Assert.Empty(missing);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindMissingGroundTruthEvaluators_IgnoresRubricRefs()
|
||||
{
|
||||
// Rubric refs are not ground-truth–dependent and must be skipped even when
|
||||
// no items carry ExpectedOutput.
|
||||
var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "1");
|
||||
var missing = FoundryEvalConverter.FindMissingGroundTruthEvaluators(
|
||||
[rubric, "relevance"], hasGroundTruth: false);
|
||||
|
||||
Assert.Empty(missing);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertMessage DataContent test
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Microsoft.Agents.AI.Foundry.UnitTests;
|
||||
|
||||
@@ -13,7 +15,7 @@ public sealed class FoundryEvalsTests
|
||||
public void FilterToolEvaluators_AllToolEvaluators_NoTools_ThrowsArgumentException()
|
||||
{
|
||||
// All configured evaluators are tool-type, but no items have tools.
|
||||
var evaluators = new[] { "tool_call_accuracy", "tool_selection" };
|
||||
var evaluators = new FoundryEvaluatorSpec[] { "tool_call_accuracy", "tool_selection" };
|
||||
|
||||
var ex = Assert.Throws<ArgumentException>(
|
||||
() => FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false));
|
||||
@@ -24,23 +26,229 @@ public sealed class FoundryEvalsTests
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_MixedEvaluators_NoTools_FiltersToolOnes()
|
||||
{
|
||||
var evaluators = new[] { "relevance", "tool_call_accuracy", "coherence" };
|
||||
var evaluators = new FoundryEvaluatorSpec[] { "relevance", "tool_call_accuracy", "coherence" };
|
||||
|
||||
var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false);
|
||||
|
||||
Assert.Equal(2, result.Length);
|
||||
Assert.Contains("relevance", result);
|
||||
Assert.Contains("coherence", result);
|
||||
Assert.DoesNotContain("tool_call_accuracy", result);
|
||||
Assert.Contains((FoundryEvaluatorSpec)"relevance", result);
|
||||
Assert.Contains((FoundryEvaluatorSpec)"coherence", result);
|
||||
Assert.DoesNotContain((FoundryEvaluatorSpec)"tool_call_accuracy", result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_HasTools_ReturnsAllEvaluators()
|
||||
{
|
||||
var evaluators = new[] { "relevance", "tool_call_accuracy" };
|
||||
var evaluators = new FoundryEvaluatorSpec[] { "relevance", "tool_call_accuracy" };
|
||||
|
||||
var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: true);
|
||||
|
||||
Assert.Equal(evaluators, result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilterToolEvaluators_PreservesRubricRefs_WhenNoTools()
|
||||
{
|
||||
// Rubric refs are tool-aware but never tool-required, so they must survive filtering
|
||||
// when no items carry tool definitions.
|
||||
var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "3");
|
||||
var evaluators = new FoundryEvaluatorSpec[] { "relevance", rubric, "tool_call_accuracy" };
|
||||
|
||||
var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false);
|
||||
|
||||
Assert.Equal(2, result.Length);
|
||||
Assert.Contains((FoundryEvaluatorSpec)"relevance", result);
|
||||
Assert.Contains(result, s => s.IsRubric && s.GeneratedRef!.Name == "policy-rubric");
|
||||
Assert.DoesNotContain((FoundryEvaluatorSpec)"tool_call_accuracy", result);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvals.ParseRubricScores tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ParseRubricScores_CanonicalDimensionScoresKey_ParsesAllFields()
|
||||
{
|
||||
// Per Microsoft Learn docs, runtime output uses properties.dimension_scores.
|
||||
const string Json = """
|
||||
{
|
||||
"properties": {
|
||||
"dimension_scores": [
|
||||
{ "id": "intent_recognition", "score": 5, "applicable": true, "weight": 9, "reason": "Identified correctly." },
|
||||
{ "id": "general_quality", "score": 4, "applicable": true, "weight": 5, "reason": "Strong overall." }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
using var doc = JsonDocument.Parse(Json);
|
||||
|
||||
var result = FoundryEvals.ParseRubricScores(doc.RootElement);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(2, result!.Count);
|
||||
Assert.Equal(["intent_recognition", "general_quality"], result.Select(r => r.Id));
|
||||
Assert.Equal([5, 4], result.Select(r => r.Score));
|
||||
Assert.Equal([9, 5], result.Select(r => r.Weight));
|
||||
Assert.True(result[0].Applicable);
|
||||
Assert.Equal("Identified correctly.", result[0].Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseRubricScores_LegacyRubricScoresKey_StillSupported()
|
||||
{
|
||||
// Preview builds used the rubric_scores key; we still accept it for back-compat.
|
||||
const string Json = """
|
||||
{
|
||||
"properties": {
|
||||
"rubric_scores": [
|
||||
{ "id": "a", "score": 3, "applicable": true, "weight": 1, "reason": "r" }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
using var doc = JsonDocument.Parse(Json);
|
||||
|
||||
var result = FoundryEvals.ParseRubricScores(doc.RootElement);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Single(result!);
|
||||
Assert.Equal("a", result[0].Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseRubricScores_TopLevelKey_FallsBack()
|
||||
{
|
||||
// Defensive fallback when SDK shape omits the 'properties' wrapper.
|
||||
const string Json = """
|
||||
{
|
||||
"dimension_scores": [
|
||||
{ "id": "x", "score": 2, "applicable": true, "weight": 1, "reason": "" }
|
||||
]
|
||||
}
|
||||
""";
|
||||
using var doc = JsonDocument.Parse(Json);
|
||||
|
||||
var result = FoundryEvals.ParseRubricScores(doc.RootElement);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Single(result!);
|
||||
Assert.Equal("x", result[0].Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseRubricScores_NoRubricKeys_ReturnsNull()
|
||||
{
|
||||
const string Json = """
|
||||
{ "properties": { "other_field": [] } }
|
||||
""";
|
||||
using var doc = JsonDocument.Parse(Json);
|
||||
|
||||
var result = FoundryEvals.ParseRubricScores(doc.RootElement);
|
||||
|
||||
Assert.Null(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseRubricScores_SkipsMalformedEntries()
|
||||
{
|
||||
// Entries missing weight or applicable are skipped, but well-formed siblings are kept.
|
||||
const string Json = """
|
||||
{
|
||||
"properties": {
|
||||
"dimension_scores": [
|
||||
{ "id": "good", "score": 3, "applicable": true, "weight": 1, "reason": "ok" },
|
||||
{ "id": "bad-no-weight", "score": 2, "applicable": true, "reason": "x" },
|
||||
{ "id": "bad-no-applicable", "score": 2, "weight": 1, "reason": "x" }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
using var doc = JsonDocument.Parse(Json);
|
||||
|
||||
var result = FoundryEvals.ParseRubricScores(doc.RootElement);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Single(result!);
|
||||
Assert.Equal("good", result[0].Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseRubricScores_NonApplicableDimension_KeepsNullScoreWhenMissing()
|
||||
{
|
||||
// Non-applicable dimensions can legitimately omit score (or set it to null).
|
||||
const string Json = """
|
||||
{
|
||||
"properties": {
|
||||
"dimension_scores": [
|
||||
{ "id": "skipped", "applicable": false, "weight": 5, "reason": "n/a" }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
using var doc = JsonDocument.Parse(Json);
|
||||
|
||||
var result = FoundryEvals.ParseRubricScores(doc.RootElement);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Single(result!);
|
||||
Assert.Equal("skipped", result[0].Id);
|
||||
Assert.False(result[0].Applicable);
|
||||
Assert.Null(result[0].Score);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvaluatorSpec validation tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void FoundryEvaluatorSpec_Default_IsNotValid()
|
||||
{
|
||||
var spec = default(FoundryEvaluatorSpec);
|
||||
Assert.False(spec.IsValid);
|
||||
Assert.Null(spec.BuiltinName);
|
||||
Assert.Null(spec.GeneratedRef);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FoundryEvaluatorSpec_EnsureValid_DefaultThrows()
|
||||
{
|
||||
var spec = default(FoundryEvaluatorSpec);
|
||||
var ex = Assert.Throws<ArgumentException>(() => spec.EnsureValid("evaluators"));
|
||||
Assert.Equal("evaluators", ex.ParamName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FoundryEvaluatorSpec_EnsureValid_BuiltinPasses()
|
||||
{
|
||||
var spec = (FoundryEvaluatorSpec)"relevance";
|
||||
spec.EnsureValid(); // does not throw
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FoundryEvaluatorSpec_EnsureValid_RubricPasses()
|
||||
{
|
||||
var spec = (FoundryEvaluatorSpec)new GeneratedEvaluatorRef("r", "1");
|
||||
spec.EnsureValid(); // does not throw
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureAllSpecsValid_DefaultEntry_ThrowsWithParamName()
|
||||
{
|
||||
var specs = new FoundryEvaluatorSpec[] { "relevance", default };
|
||||
var ex = Assert.Throws<ArgumentException>(
|
||||
() => FoundryEvals.EnsureAllSpecsValid(specs, "evaluators"));
|
||||
Assert.Equal("evaluators", ex.ParamName);
|
||||
Assert.Contains("index 1", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureAllSpecsValid_AllValid_DoesNotThrow()
|
||||
{
|
||||
var specs = new FoundryEvaluatorSpec[]
|
||||
{
|
||||
"relevance",
|
||||
new GeneratedEvaluatorRef("policy", "1"),
|
||||
};
|
||||
FoundryEvals.EnsureAllSpecsValid(specs, "evaluators");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ public class HarnessAgentOptionsTests
|
||||
Assert.Null(options.AIContextProviders);
|
||||
Assert.Null(options.LoopEvaluators);
|
||||
Assert.Null(options.LoopAgentOptions);
|
||||
Assert.False(options.DisableToolApproval);
|
||||
Assert.False(options.DisableToolAutoApproval);
|
||||
Assert.False(options.DisableNonApprovalRequiredFunctionBypassing);
|
||||
Assert.False(options.DisableFileMemory);
|
||||
Assert.False(options.DisableFileAccess);
|
||||
@@ -85,7 +85,7 @@ public class HarnessAgentOptionsTests
|
||||
ChatHistoryProvider = chatHistoryProvider,
|
||||
AIContextProviders = contextProviders,
|
||||
MaximumIterationsPerRequest = 42,
|
||||
DisableToolApproval = true,
|
||||
DisableToolAutoApproval = true,
|
||||
DisableNonApprovalRequiredFunctionBypassing = true,
|
||||
DisableFileMemory = true,
|
||||
FileMemoryStore = fileMemoryStore,
|
||||
@@ -120,7 +120,7 @@ public class HarnessAgentOptionsTests
|
||||
Assert.Same(chatHistoryProvider, options.ChatHistoryProvider);
|
||||
Assert.Same(contextProviders, options.AIContextProviders);
|
||||
Assert.Equal(42, options.MaximumIterationsPerRequest);
|
||||
Assert.True(options.DisableToolApproval);
|
||||
Assert.True(options.DisableToolAutoApproval);
|
||||
Assert.True(options.DisableNonApprovalRequiredFunctionBypassing);
|
||||
Assert.True(options.DisableFileMemory);
|
||||
Assert.Same(fileMemoryStore, options.FileMemoryStore);
|
||||
|
||||
@@ -27,7 +27,7 @@ public class HarnessAgentTests
|
||||
{
|
||||
MaxContextWindowTokens = TestMaxContextWindowTokens,
|
||||
MaxOutputTokens = TestMaxOutputTokens,
|
||||
DisableToolApproval = true,
|
||||
DisableToolAutoApproval = true,
|
||||
DisableOpenTelemetry = true,
|
||||
DisableFileMemory = true,
|
||||
DisableFileAccess = true,
|
||||
@@ -626,7 +626,7 @@ public class HarnessAgentTests
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var options = CreateAllDisabledOptions();
|
||||
options.DisableToolApproval = false;
|
||||
options.DisableToolAutoApproval = false;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, options);
|
||||
@@ -679,7 +679,7 @@ public class HarnessAgentTests
|
||||
});
|
||||
|
||||
var options = CreateAllDisabledOptions();
|
||||
options.DisableToolApproval = false;
|
||||
options.DisableToolAutoApproval = false;
|
||||
options.ToolApprovalAgentOptions = new ToolApprovalAgentOptions
|
||||
{
|
||||
AutoApprovalRules = [fcc => new ValueTask<bool>(fcc.Name == "ReadTool")]
|
||||
@@ -1695,7 +1695,7 @@ public class HarnessAgentTests
|
||||
{
|
||||
MaxContextWindowTokens = TestMaxContextWindowTokens,
|
||||
MaxOutputTokens = TestMaxOutputTokens,
|
||||
DisableToolApproval = true,
|
||||
DisableToolAutoApproval = true,
|
||||
DisableOpenTelemetry = true,
|
||||
DisableFileMemory = true,
|
||||
DisableFileAccess = true,
|
||||
@@ -1746,7 +1746,7 @@ public class HarnessAgentTests
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var options = new HarnessAgentOptions
|
||||
{
|
||||
DisableToolApproval = true,
|
||||
DisableToolAutoApproval = true,
|
||||
DisableOpenTelemetry = true,
|
||||
DisableFileMemory = true,
|
||||
DisableFileAccess = true,
|
||||
@@ -1778,7 +1778,7 @@ public class HarnessAgentTests
|
||||
var options = new HarnessAgentOptions
|
||||
{
|
||||
MaxContextWindowTokens = TestMaxContextWindowTokens,
|
||||
DisableToolApproval = true,
|
||||
DisableToolAutoApproval = true,
|
||||
DisableOpenTelemetry = true,
|
||||
DisableFileMemory = true,
|
||||
DisableFileAccess = true,
|
||||
@@ -1887,7 +1887,7 @@ public class HarnessAgentTests
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var options = CreateAllDisabledOptions();
|
||||
options.DisableToolApproval = false;
|
||||
options.DisableToolAutoApproval = false;
|
||||
options.LoopEvaluators = [new DelegateLoopEvaluator((_, _) => new ValueTask<LoopEvaluation>(LoopEvaluation.Stop()))];
|
||||
|
||||
// Act
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Microsoft.Agents.AI.Hosting.AzureFunctions.UnitTests;
|
||||
|
||||
public sealed class BuiltInFunctionsWorkflowRoutingTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("http-MyWorkflow-status", "-status", "MyWorkflow")]
|
||||
[InlineData("http-OrderProcessor-status", "-status", "OrderProcessor")]
|
||||
[InlineData("http-MyWorkflow-respond", "-respond", "MyWorkflow")]
|
||||
[InlineData("http-Multi-Dash-Name-status", "-status", "Multi-Dash-Name")]
|
||||
public void GetWorkflowName_ReturnsCorrectName(string functionName, string suffix, string expectedWorkflowName)
|
||||
{
|
||||
// Act
|
||||
string result = BuiltInFunctions.GetWorkflowName(functionName, suffix);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(expectedWorkflowName, result);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("invalid-name", "-status")]
|
||||
[InlineData("http-MyWorkflow-respond", "-status")] // wrong suffix
|
||||
[InlineData("mcptool-MyWorkflow-status", "-status")] // wrong prefix
|
||||
public void GetWorkflowName_ThrowsForInvalidPattern(string functionName, string suffix)
|
||||
{
|
||||
// Act & Assert
|
||||
Assert.Throws<InvalidOperationException>(() =>
|
||||
BuiltInFunctions.GetWorkflowName(functionName, suffix));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("dafx-MyWorkflow", "http-MyWorkflow-status", "-status", true)]
|
||||
[InlineData("dafx-MyWorkflow", "http-MyWorkflow-respond", "-respond", true)]
|
||||
[InlineData("dafx-myworkflow", "http-MyWorkflow-status", "-status", true)] // case-insensitive
|
||||
[InlineData("dafx-MYWORKFLOW", "http-MyWorkflow-respond", "-respond", true)] // case-insensitive
|
||||
[InlineData("dafx-OtherWorkflow", "http-MyWorkflow-status", "-status", false)] // cross-workflow
|
||||
[InlineData("dafx-PrivilegedWorkflow", "http-PublicWorkflow-status", "-status", false)] // attack scenario
|
||||
[InlineData("dafx-PrivilegedWorkflow", "http-PublicWorkflow-respond", "-respond", false)] // attack scenario
|
||||
public void IsOrchestrationOwnedByWorkflow_ValidatesCorrectly(
|
||||
string orchestrationName,
|
||||
string functionName,
|
||||
string suffix,
|
||||
bool expectedResult)
|
||||
{
|
||||
// Act
|
||||
bool result = BuiltInFunctions.IsOrchestrationOwnedByWorkflow(orchestrationName, functionName, suffix);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(expectedResult, result);
|
||||
}
|
||||
}
|
||||
+22
-2
@@ -98,6 +98,24 @@ public sealed class HostedAgentSkillsPatternTests : IDisposable
|
||||
Assert.True(Directory.Exists(Path.Combine(destDir, "subdir")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SafeExtractZip_NestedFileWithinDestination_Extracts()
|
||||
{
|
||||
// Arrange — a legitimate nested entry must still pass the single containment gate
|
||||
string destDir = Path.Combine(this._testRoot, "nested-extract");
|
||||
Directory.CreateDirectory(destDir);
|
||||
byte[] zip = CreateZipWithEntry("docs/SKILL.md", "nested body");
|
||||
|
||||
// Act
|
||||
using var archive = new ZipArchive(new MemoryStream(zip), ZipArchiveMode.Read);
|
||||
SafeExtractZip(archive, destDir);
|
||||
|
||||
// Assert
|
||||
string extracted = Path.Combine(destDir, "docs", "SKILL.md");
|
||||
Assert.True(File.Exists(extracted));
|
||||
Assert.Equal("nested body", File.ReadAllText(extracted));
|
||||
}
|
||||
|
||||
// ── Skill name validation tests ──────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
@@ -273,9 +291,11 @@ public sealed class HostedAgentSkillsPatternTests : IDisposable
|
||||
|
||||
foreach (ZipArchiveEntry entry in archive.Entries)
|
||||
{
|
||||
// Resolve the entry against the destination, then require the result to stay within the
|
||||
// destination subtree. A single StartsWith containment check is the only gate to
|
||||
// extraction, so any entry that escapes (for example via '..') is rejected.
|
||||
string entryPath = Path.GetFullPath(Path.Combine(destRoot, entry.FullName));
|
||||
if (!entryPath.StartsWith(destRootWithSep, comparison)
|
||||
&& !string.Equals(entryPath, destRoot, comparison))
|
||||
if (!entryPath.StartsWith(destRootWithSep, comparison))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Refusing to extract unsafe path '{entry.FullName}' outside of '{destRoot}'.");
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user