Python: Clean up dependency groups and compatibility (#7046)

* Python: Clean up dependency management Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2f7b1c89-f3ff-418d-ab4e-4f014fda308f * Python: Harden Mistral SDK import fallback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2f7b1c89-f3ff-418d-ab4e-4f014fda308f
2026-07-11 00:40:42 +02:00
parent 875031ff56
commit 68136ee081
53 changed files with 1598 additions and 1505 deletions
@@ -119,7 +119,7 @@ class PackagePlan:
    package_name: str
    pyproject_path: Path
    internal_editables: list[Path]
-    include_dev_group: bool
+    dependency_groups: list[str]
    include_dev_extra: bool
    optional_extras: list[str]

@@ -610,7 +610,7 @@ def _run_tasks(
    internal_editables: list[Path],
    resolution: str,
    dependency_pin: tuple[str, Version] | None,
-    include_dev_group: bool,
+    dependency_groups: list[str],
    include_dev_extra: bool,
    optional_extras: list[str],
    timeout_seconds: int,
@@ -637,8 +637,8 @@ def _run_tasks(
            "--quiet",
        ]
        extend_command_with_runtime_tools(command, workspace_root)
-        if include_dev_group:
-            command.extend(["--group", "dev"])
+        for group_name in dependency_groups:
+            command.extend(["--group", group_name])
        if include_dev_extra:
            command.extend(["--extra", "dev"])
        for extra_name in optional_extras:
@@ -679,7 +679,7 @@ def _optimize_dependency(
    max_candidates: int,
    timeout_seconds: int,
    package_label: str,
-    include_dev_group: bool,
+    dependency_groups: list[str],
    include_dev_extra: bool,
    optional_extras: list[str],
 ) -> DependencyOutcome:
@@ -720,7 +720,7 @@ def _optimize_dependency(
        internal_editables=internal_editables,
        resolution="lowest-direct",
        dependency_pin=(dependency.name, baseline_version),
-        include_dev_group=include_dev_group,
+        dependency_groups=dependency_groups,
        include_dev_extra=include_dev_extra,
        optional_extras=optional_extras,
        timeout_seconds=timeout_seconds,
@@ -779,7 +779,7 @@ def _optimize_dependency(
            internal_editables=internal_editables,
            resolution="lowest-direct",
            dependency_pin=(dependency.name, candidate),
-            include_dev_group=include_dev_group,
+            dependency_groups=dependency_groups,
            include_dev_extra=include_dev_extra,
            optional_extras=optional_extras,
            timeout_seconds=timeout_seconds,
@@ -902,7 +902,7 @@ def _process_package(
                max_candidates=max_candidates,
                timeout_seconds=timeout_seconds,
                package_label=package_label,
-                include_dev_group=plan.include_dev_group,
+                dependency_groups=plan.dependency_groups,
                include_dev_extra=plan.include_dev_extra,
                optional_extras=plan.optional_extras,
            )
@@ -1053,7 +1053,7 @@ def main() -> None:
                package_name=package_name,
                pyproject_path=pyproject_file,
                internal_editables=_resolve_internal_editables(package_name, package_map, internal_graph),
-                include_dev_group="dev" in dependency_groups,
+                dependency_groups=sorted(dependency_groups),
                include_dev_extra="dev" in optional_dependencies,
                optional_extras=sorted(name for name in optional_dependencies if name not in {"all", "dev"}),
            )
@@ -129,7 +129,7 @@ class PackagePlan:
    package_name: str
    pyproject_path: Path
    internal_editables: list[Path]
-    include_dev_group: bool
+    dependency_groups: list[str]
    include_dev_extra: bool
    optional_extras: list[str]

@@ -252,7 +252,7 @@ def _exact_pin_version(requirement: Requirement) -> Version | None:
    return None


-def _collect_dev_pin_replacements(
+def _collect_development_pin_replacements(
    pyproject_file: Path,
    *,
    catalog: VersionCatalog,
@@ -263,35 +263,36 @@ def _collect_dev_pin_replacements(
    optional_dependencies = project.get("optional-dependencies", {}) or {}
    dependency_groups = data.get("dependency-groups", {}) or {}
    logger.debug(
-        "Collecting dev dependency replacements from %s with optional_dependencies=%s and dependency_groups=%s",
+        "Collecting development dependency replacements from %s with optional_dependencies=%s and dependency_groups=%s",
        pyproject_file,
        optional_dependencies.keys(),
        dependency_groups.keys(),
    )
-    dev_requirements: list[str] = []
-    dev_requirements.extend(
+    development_requirements: list[str] = []
+    development_requirements.extend(
        requirement for requirement in (optional_dependencies.get("dev", []) or []) if isinstance(requirement, str)
    )
-    dev_requirements.extend(
-        requirement for requirement in (dependency_groups.get("dev", []) or []) if isinstance(requirement, str)
-    )
-    logger.debug(f"Found {len(dev_requirements)} dev requirements in {pyproject_file}")
-    parsed_dev_requirements: dict[str, Requirement] = {}
-    for requirement in dev_requirements:
+    for group_requirements in dependency_groups.values():
+        development_requirements.extend(
+            requirement for requirement in (group_requirements or []) if isinstance(requirement, str)
+        )
+    logger.debug(f"Found {len(development_requirements)} development requirements in {pyproject_file}")
+    parsed_development_requirements: dict[str, Requirement] = {}
+    for requirement in development_requirements:
        try:
            parsed_requirement = Requirement(requirement)
        except InvalidRequirement:
            continue
-        parsed_dev_requirements[parsed_requirement.name.lower()] = parsed_requirement
+        parsed_development_requirements[parsed_requirement.name.lower()] = parsed_requirement

    seen_requirements: set[str] = set()
    replacements: dict[str, str] = {}
-    for requirement in dev_requirements:
+    for requirement in development_requirements:
        if requirement in seen_requirements:
            continue
        seen_requirements.add(requirement)

-        # Refresh exact dev pins while we already have the file open so outdated test tooling
+        # Refresh exact development pins while we already have the file open so outdated test tooling
        # does not masquerade as a runtime dependency compatibility failure.
        try:
            parsed_requirement = Requirement(requirement)
@@ -314,18 +315,16 @@ def _collect_dev_pin_replacements(
        if latest_version is None:
            continue
        current_exact_version = _exact_pin_version(parsed_requirement)
-        if current_exact_version is not None and dependency_name in VALIDATION_TOOL_DEV_PINS:
+        if current_exact_version is None:
+            continue
+        if dependency_name in VALIDATION_TOOL_DEV_PINS:
            logger.info(
                "Skipping %s in %s because validation tool upgrades should be handled separately.",
                dependency_name,
                pyproject_file,
            )
            continue
-        if current_exact_version is None:
-            locked_version = _select_latest_dev_version(catalog.get_lock(dependency_name))
-            if locked_version is not None:
-                latest_version = locked_version
-        if current_exact_version is not None and latest_version < current_exact_version:
+        if latest_version < current_exact_version:
            logger.info(
                "Skipping %s in %s because selected version %s is older than current pin %s.",
                dependency_name,
@@ -336,8 +335,7 @@ def _collect_dev_pin_replacements(
            continue
        if (
            dependency_name == OPENTELEMETRY_SDK
-            and AZURE_MONITOR_OPENTELEMETRY in parsed_dev_requirements
-            and current_exact_version is not None
+            and AZURE_MONITOR_OPENTELEMETRY in parsed_development_requirements
            and latest_version != current_exact_version
        ):
            logger.info(
@@ -776,9 +774,7 @@ def _select_upper_probe_version(
    allow_prerelease: bool,
 ) -> Version | None:
    """Return the newest concrete version that would be allowed by a candidate upper bound."""
-    probe_versions = [
-        version for version in versions if version < upper_bound and (lower is None or version >= lower)
-    ]
+    probe_versions = [version for version in versions if version < upper_bound and (lower is None or version >= lower)]
    if not allow_prerelease:
        probe_versions = [version for version in probe_versions if not version.is_prerelease]
    return probe_versions[-1] if probe_versions else None
@@ -792,7 +788,7 @@ def _run_tasks(
    internal_editables: list[Path],
    resolution: str,
    dependency_pin: tuple[str, Version] | None,
-    include_dev_group: bool,
+    dependency_groups: list[str],
    include_dev_extra: bool,
    optional_extras: list[str],
    timeout_seconds: int,
@@ -818,8 +814,8 @@ def _run_tasks(
            "--quiet",
        ]
        extend_command_with_runtime_tools(command, workspace_root)
-        if include_dev_group:
-            command.extend(["--group", "dev"])
+        for group_name in dependency_groups:
+            command.extend(["--group", group_name])
        if include_dev_extra:
            command.extend(["--extra", "dev"])
        for extra_name in optional_extras:
@@ -860,7 +856,7 @@ def _optimize_dependency(
    max_candidates: int,
    timeout_seconds: int,
    package_label: str,
-    include_dev_group: bool,
+    dependency_groups: list[str],
    include_dev_extra: bool,
    optional_extras: list[str],
 ) -> DependencyOutcome:
@@ -914,7 +910,7 @@ def _optimize_dependency(
            internal_editables=internal_editables,
            resolution=baseline_resolution,
            dependency_pin=(dependency.name, baseline_version),
-            include_dev_group=include_dev_group,
+            dependency_groups=dependency_groups,
            include_dev_extra=include_dev_extra,
            optional_extras=optional_extras,
            timeout_seconds=timeout_seconds,
@@ -985,7 +981,7 @@ def _optimize_dependency(
            internal_editables=internal_editables,
            resolution="highest",
            dependency_pin=(dependency.name, probe_version),
-            include_dev_group=include_dev_group,
+            dependency_groups=dependency_groups,
            include_dev_extra=include_dev_extra,
            optional_extras=optional_extras,
            timeout_seconds=timeout_seconds,
@@ -1073,17 +1069,18 @@ def _process_package(
            if candidate.exists():
                temp_internal_editables.append(candidate)

-        dev_replacements = _collect_dev_pin_replacements(temp_pyproject, catalog=catalog)
-        if dev_replacements:
-            _replace_requirements(temp_pyproject, list(dev_replacements.items()))
+        development_replacements = _collect_development_pin_replacements(temp_pyproject, catalog=catalog)
+        if development_replacements:
+            _replace_requirements(temp_pyproject, list(development_replacements.items()))
            print(
-                f"[cyan]{plan.project_path}: refreshed {len(dev_replacements)} dev dependency pin(s) to latest[/cyan]"
+                f"[cyan]{plan.project_path}: refreshed "
+                f"{len(development_replacements)} development dependency pin(s) to latest[/cyan]"
            )

        targets, skipped = _collect_targets(temp_pyproject, dependency_filters=dependency_filters)

        dependency_results: list[DependencyOutcome] = []
-        replacements: dict[str, str] = dict(dev_replacements)
+        replacements: dict[str, str] = dict(development_replacements)
        package_label = f"{plan.project_path} ({plan.package_name})"

        if not targets:
@@ -1102,7 +1099,7 @@ def _process_package(
                max_candidates=max_candidates,
                timeout_seconds=timeout_seconds,
                package_label=package_label,
-                include_dev_group=plan.include_dev_group,
+                dependency_groups=plan.dependency_groups,
                include_dev_extra=plan.include_dev_extra,
                optional_extras=plan.optional_extras,
            )
@@ -1254,7 +1251,7 @@ def main() -> None:
                package_name=package_name,
                pyproject_path=pyproject_file,
                internal_editables=_resolve_internal_editables(package_name, package_map, internal_graph),
-                include_dev_group="dev" in dependency_groups,
+                dependency_groups=sorted(dependency_groups),
                include_dev_extra="dev" in optional_dependencies,
                optional_extras=sorted(name for name in optional_dependencies if name not in {"all", "dev"}),
            )
@@ -1279,7 +1276,7 @@ def main() -> None:
                package_name=root_package_name,
                pyproject_path=workspace_pyproject,
                internal_editables=[],
-                include_dev_group="dev" in root_dependency_groups,
+                dependency_groups=sorted(root_dependency_groups),
                include_dev_extra="dev" in root_optional_dependencies,
                optional_extras=sorted(name for name in root_optional_dependencies if name not in {"all", "dev"}),
            )
@@ -1,12 +1,11 @@
 # Copyright (c) Microsoft. All rights reserved.
-# ruff: noqa: INP001

-"""Refresh dev dependency pins across the Python workspace."""
+"""Refresh development dependency pins across the Python workspace."""

 from __future__ import annotations

-import logging
 import argparse
+import logging
 from dataclasses import dataclass
 from pathlib import Path

@@ -16,16 +15,17 @@ from rich import print
 from scripts.dependencies._dependency_bounds_upper_impl import (
    VersionCatalog,
    _apply_package_replacements,
-    _collect_dev_pin_replacements,
+    _collect_development_pin_replacements,
    _load_lock_versions,
 )
 from scripts.task_runner import discover_projects

 logger = logging.getLogger(__name__)

+
@dataclass(frozen=True)
 class WorkspaceProject:
-    """Workspace project metadata used for dev dependency pin refresh."""
+    """Workspace project metadata used for development dependency pin refresh."""

    name: str
    project_path: str
@@ -56,7 +56,7 @@ def _discover_workspace_projects(workspace_root: Path) -> list[WorkspaceProject]
    ]

    # The root project carries the repo-wide dev toolchain pins, while package pyprojects may
-    # carry package-specific dev extras/groups. Refresh both surfaces in one pass so the
+    # carry package-specific development groups. Refresh both surfaces in one pass so the
    # workspace stays internally consistent after a tooling bump.
    # Reuse the shared workspace discovery logic so this script stays aligned with the rest
    # of the repo-level task runners when packages are added or moved.
@@ -103,10 +103,10 @@ def _select_projects(projects: list[WorkspaceProject], package_filters: list[str


 def main() -> None:
-    """Refresh exact dev dependency pins in workspace pyproject files."""
+    """Refresh exact development dependency pins in workspace pyproject files."""
    parser = argparse.ArgumentParser(
        description=(
-            "Refresh dev dependency pins across the workspace pyproject.toml files. "
+            "Refresh development dependency pins across the workspace pyproject.toml files. "
            "By default, resolves versions from PyPI and falls back to uv.lock when network access is unavailable."
        )
    )
@@ -120,7 +120,7 @@ def main() -> None:
        "--version-source",
        choices=["pypi", "lock"],
        default="pypi",
-        help="Version source for selecting the newest dev pin.",
+        help="Version source for selecting the newest development dependency pin.",
    )
    parser.add_argument(
        "--dry-run",
@@ -137,7 +137,7 @@ def main() -> None:

    workspace_root = Path(__file__).resolve().parents[2]
    lock_versions = _load_lock_versions(workspace_root)
-    # Reuse the same version catalog as the bound-expansion tooling so dev pin refreshes choose
+    # Reuse the same version catalog as the bound-expansion tooling so development pin refreshes choose
    # versions with the same PyPI-vs-lock fallback behavior as the dependency validators.
    catalog = VersionCatalog(lock_versions=lock_versions, source=args.version_source)

@@ -145,7 +145,10 @@ def main() -> None:
        _discover_workspace_projects(workspace_root),
        package_filters=args.packages,
    )
-    logger.debug(f"Selected projects for dev dependency refresh: {[project.pyproject_path for project in selected_projects]}")
+    logger.debug(
+        "Selected projects for development dependency refresh: %s",
+        [project.pyproject_path for project in selected_projects],
+    )
    if not selected_projects:
        filters = ", ".join(args.packages or [])
        raise SystemExit(f"No matching workspace projects found for: {filters}")
@@ -153,10 +156,10 @@ def main() -> None:
    updated_projects = 0
    updated_requirements = 0
    for project in selected_projects:
-        # Keep the replacement logic centralized in the upper-bound helper so exact dev pins are
+        # Keep the replacement logic centralized in the upper-bound helper so exact development pins are
        # formatted consistently regardless of whether we update them directly here or while
        # widening runtime dependency bounds.
-        replacements = _collect_dev_pin_replacements(project.pyproject_file, catalog=catalog)
+        replacements = _collect_development_pin_replacements(project.pyproject_file, catalog=catalog)
        if not replacements:
            continue

@@ -171,16 +174,16 @@ def main() -> None:
        _apply_package_replacements(project.pyproject_file, replacements)
        print(
            f"[green]Updated {project.pyproject_path}[/green] "
-            f"({project.name}) with {len(replacements)} dev dependency pin refresh(es)."
+            f"({project.name}) with {len(replacements)} development dependency pin refresh(es)."
        )

    if updated_projects == 0:
-        print("[green]No dev dependency pin updates were needed.[/green]")
+        print("[green]No development dependency pin updates were needed.[/green]")
        return

    action = "Would update" if args.dry_run else "Updated"
    print(
-        f"[green]{action} {updated_requirements} dev dependency pin(s) "
+        f"[green]{action} {updated_requirements} development dependency pin(s) "
        f"across {updated_projects} workspace project(s).[/green]"
    )

@@ -50,7 +50,7 @@ class PackageTestPlan:

    project_path: Path
    package_name: str
-    include_dev_group: bool
+    dependency_groups: list[str]
    include_dev_extra: bool
    optional_extras: list[str]
    internal_editables: list[Path]
@@ -120,7 +120,7 @@ def _build_test_plans(workspace_root: Path, package_filter: str | None) -> list[
            PackageTestPlan(
                project_path=project_path,
                package_name=package_name,
-                include_dev_group="dev" in dependency_groups,
+                dependency_groups=sorted(dependency_groups),
                include_dev_extra="dev" in optional_dependencies,
                optional_extras=sorted(name for name in optional_dependencies if name not in {"all", "dev"}),
                internal_editables=_resolve_internal_editables(package_name, package_map, internal_graph),
@@ -164,8 +164,8 @@ def _run_package_tasks(
            "--quiet",
        ]
        extend_command_with_runtime_tools(command, workspace_root)
-        if plan.include_dev_group:
-            command.extend(["--group", "dev"])
+        for group_name in plan.dependency_groups:
+            command.extend(["--group", group_name])
        if plan.include_dev_extra:
            command.extend(["--extra", "dev"])
        for extra_name in plan.optional_extras:
@@ -257,14 +257,12 @@ def _run_test_mode(
                timeout_seconds=timeout_seconds,
                dry_run=dry_run,
            )
-            scenario_result["packages"].append(
-                {
-                    "project_path": str(plan.project_path),
-                    "package_name": plan.package_name,
-                    "status": "passed" if success else "failed",
-                    "error": error,
-                }
-            )
+            scenario_result["packages"].append({
+                "project_path": str(plan.project_path),
+                "package_name": plan.package_name,
+                "status": "passed" if success else "failed",
+                "error": error,
+            })
            if success:
                print(f"[green]{plan.project_path}: {scenario_name} passed[/green]")
                continue