microsoft--agent-framework/.github/workflows/python-sample-validation.yml

name: Python - Sample Validation

on:
  workflow_dispatch:
  schedule:
    - cron: "0 0 * * *" # Run at midnight UTC daily

env:
  # Configure a constant location for the uv cache
  UV_CACHE_DIR: /tmp/.uv-cache
  GITHUB_TOKEN: ${{ github.token }}
  GITHUB_COPILOT_MODEL: auto

permissions:
  copilot-requests: write
  contents: read
  id-token: write

jobs:
  validate-01-get-started:
    name: Validate 01-get-started
    runs-on: ubuntu-latest
    environment: integration
    env:
      # Required configuration for get-started samples
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-01-get-started
          path: python/samples/sample_validation/reports/

  validate-02-agents:
    name: Validate 02-agents
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      # Foundry configuration
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      # Azure OpenAI configuration
      AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
      AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
      AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
      AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
      AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
      # OpenAI configuration
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
      # GitHub MCP
      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
      OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
      # Observability
      ENABLE_INSTRUMENTATION: "true"
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
          echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
          echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
          echo "AZURE_OPENAI_CHAT_COMPLETION_MODEL=$AZURE_OPENAI_CHAT_COMPLETION_MODEL" >> .env
          echo "AZURE_OPENAI_CHAT_MODEL=$AZURE_OPENAI_CHAT_MODEL" >> .env
          echo "AZURE_OPENAI_EMBEDDING_MODEL=$AZURE_OPENAI_EMBEDDING_MODEL" >> .env
          echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
          echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
          echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env
          echo "GITHUB_PAT=$GITHUB_PAT" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents
          path: python/samples/sample_validation/reports/

  validate-02-agents-harness:
    name: Validate 02-agents/harness
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      # Optional: enables the Foundry memory path in harness samples
      FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }}
      FOUNDRY_MEMORY_STORE: ${{ vars.FOUNDRY_MEMORY_STORE || '' }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
          echo "FOUNDRY_EMBEDDING_MODEL=$FOUNDRY_EMBEDDING_MODEL" >> .env
          echo "FOUNDRY_MEMORY_STORE=$FOUNDRY_MEMORY_STORE" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-harness
          path: python/samples/sample_validation/reports/

  validate-02-agents-tools:
    name: Validate 02-agents/tools
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-tools
          path: python/samples/sample_validation/reports/

  validate-02-agents-openai:
    name: Validate 02-agents/providers/openai
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      OPENAI_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
          echo "OPENAI_MODEL=$OPENAI_MODEL" >> .env
          echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
          echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-openai
          path: python/samples/sample_validation/reports/

  validate-02-agents-azure:
    name: Validate 02-agents/providers/azure
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
      AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
      AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION || '' }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
          echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
          echo "AZURE_OPENAI_API_VERSION=$AZURE_OPENAI_API_VERSION" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-azure
          path: python/samples/sample_validation/reports/

  validate-02-agents-anthropic:
    name: Validate 02-agents/providers/anthropic
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env
          echo "ANTHROPIC_CHAT_MODEL=$ANTHROPIC_CHAT_MODEL" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-anthropic
          path: python/samples/sample_validation/reports/

  validate-02-agents-github-copilot:
    name: Validate 02-agents/providers/github_copilot
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-github-copilot
          path: python/samples/sample_validation/reports/

  validate-02-agents-amazon:
    name: Validate 02-agents/providers/amazon
    if: false # Temporarily disabled - requires AWS credentials
    runs-on: ubuntu-latest
    environment: integration
    env:
      BEDROCK_CHAT_MODEL: ${{ vars.BEDROCK__CHATMODELID }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-amazon
          path: python/samples/sample_validation/reports/

  validate-02-agents-ollama:
    name: Validate 02-agents/providers/ollama
    if: false # Temporarily disabled - requires local Ollama server
    runs-on: ubuntu-latest
    environment: integration
    env:
      OLLAMA_MODEL: ${{ vars.OLLAMA__MODEL }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-ollama
          path: python/samples/sample_validation/reports/

  validate-02-agents-foundry:
    name: Validate 02-agents/providers/foundry
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME || '' }}
      FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION || '' }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
          echo "FOUNDRY_AGENT_NAME=$FOUNDRY_AGENT_NAME" >> .env
          echo "FOUNDRY_AGENT_VERSION=$FOUNDRY_AGENT_VERSION" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-foundry
          path: python/samples/sample_validation/reports/

  validate-02-agents-copilotstudio:
    name: Validate 02-agents/providers/copilotstudio
    if: false # Temporarily disabled - requires Copilot Studio setup
    runs-on: ubuntu-latest
    environment: integration
    env:
      COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
      COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
      COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }}
      COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
          echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
          echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
          echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-copilotstudio
          path: python/samples/sample_validation/reports/

  validate-02-agents-custom:
    name: Validate 02-agents/providers/custom
    runs-on: ubuntu-latest
    environment: integration
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-02-agents-custom
          path: python/samples/sample_validation/reports/

  validate-03-workflows:
    name: Validate 03-workflows
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-03-workflows
          path: python/samples/sample_validation/reports/

  validate-04-hosting-foundry-hosted-agents:
    name: Validate 04-hosting (foundry-hosted-agents)
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      # Foundry hosted agent configuration
      AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }}
      FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }}
      AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }}
      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
      TOOLBOX_ENDPOINT: ${{ vars.TOOLBOX_ENDPOINT }}
      FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_NAME }}
      MEMORY_STORE_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_MEMORY_STORE }}
      AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
      AZURE_SEARCH_INDEX_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Run sample validation
        # Maximum parallel workers is set to 1 because all samples use the same port
        run: |
          cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-04-hosting-foundry-hosted-agents
          path: python/samples/sample_validation/reports/

  validate-04-hosting-other:
    name: Validate 04-hosting (other)
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      # A2A configuration
      A2A_AGENT_HOST: http://localhost:5001/
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-04-hosting-other
          path: python/samples/sample_validation/reports/

  validate-05-end-to-end:
    name: Validate 05-end-to-end
    if: false # Temporarily disabled because of sample complexity
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      # Azure OpenAI configuration
      AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
      AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
      # Azure AI Search (for evaluation samples)
      AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
      AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }}
      AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }}
      # Evaluation sample
      FOUNDRY_MODEL_WORKFLOW: ${{ vars.FOUNDRY_MODEL_WORKFLOW || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
      FOUNDRY_MODEL_EVAL: ${{ vars.FOUNDRY_MODEL_EVAL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-05-end-to-end
          path: python/samples/sample_validation/reports/

  validate-autogen-migration:
    name: Validate autogen-migration
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      # Azure OpenAI configuration
      AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
      AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
      # OpenAI configuration
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
      OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
          echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
          echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
          echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
          echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
          echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env

      - name: Pre-install AutoGen dependencies for migration samples
        run: uv pip install "autogen-agentchat" "autogen-ext[openai]"

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration --agent-timeout 600

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-autogen-migration
          path: python/samples/sample_validation/reports/

  validate-semantic-kernel-migration:
    name: Validate semantic-kernel-migration
    if: false # Temporarily disabled - to free up Copilot quota for other jobs
    runs-on: ubuntu-latest
    environment: integration
    env:
      FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
      FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
      # Azure OpenAI configuration for AF
      AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
      AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
      # Azure OpenAI configuration for SK
      AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
      # OpenAI key
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
      OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
      OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
      # OpenAI configuration for SK
      OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
      OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
      # Copilot Studio
      COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
      COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
      COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }}
      COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }}
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup environment
        uses: ./.github/actions/sample-validation-setup
        with:
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          os: ${{ runner.os }}

      - name: Create .env for samples
        run: |
          echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
          echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
          echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
          echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
          echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
          echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
          echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env
          echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
          echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
          echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
          echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env

      - name: Run sample validation
        run: |
          cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration

      - name: Save sample playbooks
        if: ${{ !cancelled() }}
        uses: ./.github/actions/sample-validation-save-playbooks

      - name: Upload validation report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-report-semantic-kernel-migration
          path: python/samples/sample_validation/reports/

  aggregate-results:
    name: Aggregate Results
    runs-on: ubuntu-latest
    if: always()
    needs:
      - validate-01-get-started
      - validate-02-agents
      - validate-02-agents-harness
      - validate-02-agents-tools
      - validate-02-agents-openai
      - validate-02-agents-azure
      - validate-02-agents-anthropic
      - validate-02-agents-github-copilot
      - validate-02-agents-amazon
      - validate-02-agents-ollama
      - validate-02-agents-foundry
      - validate-02-agents-copilotstudio
      - validate-02-agents-custom
      - validate-03-workflows
      - validate-04-hosting-foundry-hosted-agents
      - validate-04-hosting-other
      - validate-05-end-to-end
      - validate-autogen-migration
      - validate-semantic-kernel-migration
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Download all validation reports
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: validation-report-*
          path: reports/
          merge-multiple: true

      - name: Restore validation history
        id: cache-restore
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: validation-history/
          key: validation-history-${{ github.run_id }}
          restore-keys: |
            validation-history-

      - name: Aggregate results and generate trend report
        run: |
          python3 python/scripts/sample_validation/aggregate.py \
            reports/ \
            validation-history/history.json \
            trend-report.md

      - name: Write trend report to job summary
        run: cat trend-report.md >> "$GITHUB_STEP_SUMMARY"

      - name: Save validation history
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: validation-history/
          key: validation-history-${{ github.run_id }}

      - name: Upload trend report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: validation-trend-report
          path: trend-report.md