Skip to content

Fix minimum dependency resolution against configured feeds #48487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jenny (JennyPng) wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Fix minimum dependency resolution against configured feeds #48487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
69bc60e
Fix minimum dependency resolution
JennyPng Aug 6, 2026
ff4e6aa
Re-enable Cosmos minimum dependency checks
JennyPng Aug 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 16 additions & 74 deletions eng/tools/azure-sdk-tools/ci_tools/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,8 +651,11 @@ def is_package_compatible(
if not package_requirement.specifier.contains(immutable_version):
if should_log:
logging.info(
f"Dev req dependency {package_name}'s requirement specifier of {package_requirement}"
f"is not compatible with immutable requirement {immutable_requirement}."
"Dev req dependency %s's requirement specifier of %s is not compatible with "
"immutable requirement %s.",
package_name,
package_requirement,
immutable_requirement,
)
return False

Expand Down Expand Up @@ -697,70 +700,6 @@ def get_total_coverage(
return report


def resolve_compatible_package(package_name: str, immutable_requirements: List[Requirement]) -> Optional[str]:
"""
This function attempts to resolve a compatible package version for whatever set of immutable_requirements that
the package must be compatible with.

It should only be utilized when a package is found to be incompatible with the immutable_requirements.
It will attempt to resolve the incompatibility by walking backwards through different versions of <package_name>
until a compatible version is found that works with the immutable_requirements.
"""

pypi = PyPIClient()
immovable_pkgs = {req.name: req for req in immutable_requirements}

# Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations
# package as an example.

# immovable_pkgs = the selected mindependency for azure-ai-language-conversations
# -> "azure-core==1.28.0",
# -> "isodate==0.6.1",
# -> "typing-extensions==4.0.1",
# we have the following dev_reqs for azure-ai-language-conversations
# -> ../azure-sdk-tools
# -> ../azure-identity
# -> ../azure-core

# as we walk each of the dev reqs, we check for compatibility with the immovable_packages.
# (this happens in is_package_compatible) if the dev req is incompatible, we need to resolve it.
# THIS function is what resolves it!

# since we already know that package_name is incompatible with the immovable_pkgs, we need to walk backwards
# through the versions of package_name checking to ensure that each version is compatible with the immovable_pkgs.
# if we find one that is, we will return a new requirement string for that package which will replace this dev_req line.
for pkg in immovable_pkgs:
required_package = immovable_pkgs[pkg].name
try:
required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version
except StopIteration:
required_pkg_version = None

versions = pypi.get_ordered_versions(package_name, True)
versions.reverse()

# only allow prerelease versions if the dev_req we're targeting is also prerelease
if required_pkg_version:
if not Version(required_pkg_version).is_prerelease:
versions = [v for v in versions if not v.is_prerelease]

for version in versions:
version_release = pypi.project_release(package_name, version).get("info", {}).get("requires_dist", [])

if version_release:
requirements_for_dev_req = [Requirement(r) for r in version_release]

compatible = is_package_compatible(
required_package, requirements_for_dev_req, immutable_requirements, should_log=False
)
if compatible:
# we have found a compatible version. We can return this as the new requirement line for the dev_req file.
return f"{package_name}=={version}"

# no changes necessary
return None


def handle_incompatible_minimum_dev_reqs(
setup_path: str, filtered_requirement_list: List[str], packages_for_install: List[Requirement]
) -> List[str]:
Expand All @@ -785,7 +724,6 @@ def handle_incompatible_minimum_dev_reqs(

if cleansed_dev_requirement_line:
dev_req_package = None
dev_req_version = None
requirements_for_dev_req = []

# this is a locally built wheel file, ise pkginfo to get the metadata
Expand All @@ -799,7 +737,6 @@ def handle_incompatible_minimum_dev_reqs(
local_package_metadata = pkginfo.get_metadata(cleansed_dev_requirement_line)
if local_package_metadata:
dev_req_package = local_package_metadata.name
dev_req_version = local_package_metadata.version
requirements_for_dev_req = [Requirement(r) for r in local_package_metadata.requires_dist]
else:
logging.error(
Expand All @@ -821,7 +758,6 @@ def handle_incompatible_minimum_dev_reqs(

if local_package:
dev_req_package = local_package.name
dev_req_version = local_package.version
requirements_for_dev_req = [Requirement(r) for r in local_package.requires]
else:
logging.error(
Expand All @@ -844,13 +780,19 @@ def handle_incompatible_minimum_dev_reqs(
# we understand how to parse it, so we should handle it
if dev_req_package:
if not is_package_compatible(dev_req_package, requirements_for_dev_req, packages_for_install):
new_req = resolve_compatible_package(dev_req_package, packages_for_install)

if new_req:
cleansed_reqs.append(new_req)
available_versions = PyPIClient().get_ordered_versions(dev_req_package, True)
if available_versions:
logging.info(
"Resolving incompatible local dev requirement %s from the configured index "
"against immutable requirements %s.",
dev_req_package,
packages_for_install,
)
cleansed_reqs.append(dev_req_package)
else:
logging.error(
f'Found incompatible dev requirement {dev_req_package}, but unable to locate a compatible version. Not modifying the line: "{dev_requirement_line}".'
f"Found incompatible dev requirement {dev_req_package}, but it is not published on the "
f'configured index. Not modifying the line: "{dev_requirement_line}".'
)
cleansed_reqs.append(cleansed_dev_requirement_line)
else:
Expand Down
6 changes: 3 additions & 3 deletions eng/tools/azure-sdk-tools/ci_tools/scenario/dependency_resolution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def install_dependent_packages(
with open(pkgs_file_path, "w", encoding="utf-8") as pkgs_file:
for package in released_packages:
pkgs_file.write(package + "\n")
logger.info("Created file %s to track azure packages found on PyPI", pkgs_file_path)
logger.info("Created file %s to track packages selected from the configured index", pkgs_file_path)


def check_pkg_against_overrides(pkg_specifier: str) -> List[str]:
Expand Down Expand Up @@ -239,7 +239,7 @@ def process_requirement(req: str, dependency_type: str, orig_pkg_name: str) -> s

client = PyPIClient()
versions = [str(v) for v in client.get_ordered_versions(pkg_name, True)]
logger.info("Versions available on PyPI for %s: %s", pkg_name, versions)
logger.info("Versions available from the configured index for %s: %s", pkg_name, versions)

# prepass filter before choosing a latest or minimum, eliminate prerelease versions if they are not allowed based on the specifier
if not allows_prereleases:
Expand All @@ -263,7 +263,7 @@ def process_requirement(req: str, dependency_type: str, orig_pkg_name: str) -> s
)
return pkg_name + "==" + version

logger.error("No version is found on PyPI for package %s that matches specifier %s", pkg_name, spec)
logger.error("No version is found on the configured index for package %s that matches specifier %s", pkg_name, spec)
return ""


Expand Down
77 changes: 76 additions & 1 deletion eng/tools/azure-sdk-tools/pypi_tools/azdo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
import base64
from html.parser import HTMLParser
import json
import logging
import os
import re
import sys
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from urllib.parse import unquote, urlparse

from packaging.specifiers import InvalidSpecifier, SpecifierSet
from packaging.utils import InvalidSdistFilename, InvalidWheelFilename, parse_sdist_filename, parse_wheel_filename
from packaging.version import Version, InvalidVersion, parse
from urllib3 import PoolManager, Retry

Expand All @@ -15,6 +19,18 @@ def pep503_normalize(name: str) -> str:
return re.sub(r"[-_.]+", "-", name).lower()


class _SimpleIndexParser(HTMLParser):
"""Collect package-file links and their PEP 503 metadata."""

def __init__(self):
super().__init__()
self.links: List[Dict[str, Optional[str]]] = []

def handle_starttag(self, tag: str, attrs: List[tuple[str, Optional[str]]]) -> None:
if tag == "a":
self.links.append(dict(attrs))


@dataclass(frozen=True)
class AzureArtifactsFeedConfig:
organization: str
Expand Down Expand Up @@ -167,6 +183,65 @@ def get_ordered_versions(self, package_name: str, include_deleted: bool = False)
out.sort()
return out

@staticmethod
def _version_from_link(href: str) -> Optional[Version]:
"""Extract a normalized version from a wheel or source archive link."""
filename = os.path.basename(urlparse(href).path)
filename = unquote(filename)
try:
if filename.endswith(".whl"):
return parse_wheel_filename(filename)[1]
return parse_sdist_filename(filename)[1]
except (InvalidSdistFilename, InvalidWheelFilename):
return None

def get_python_compatible_versions(self, package_name: str) -> List[Version]:
"""Return feed versions with at least one file compatible with this interpreter.

Azure Artifacts' package REST response lists versions but does not include
``Requires-Python``. The PEP 503 simple index exposes that constraint on
each distribution link, allowing minimum-dependency selection to skip
releases that cannot run on the current interpreter.

:param str package_name: The package whose versions should be listed.
:return: Sorted versions compatible with the running Python interpreter.
:rtype: list[packaging.version.Version]
"""
package = pep503_normalize(package_name)
url = f"{self._pkgs_base_url}/{self._path_prefix()}" f"/_packaging/{self._cfg.feed}/pypi/simple/{package}/"
response = self._http.request("GET", url, headers={"Accept": "text/html", **self._auth_header()})
parser = _SimpleIndexParser()
parser.feed(response.data.decode("utf-8"))

compatible_versions = set()
current_python = Version(".".join(map(str, sys.version_info[:3])))
for link in parser.links:
href = link.get("href")
if not href or "data-yanked" in link:
continue

# Compatibility is file-specific: one release can publish several
# distributions with different Python constraints.
requires_python = link.get("data-requires-python")
if requires_python:
try:
if current_python not in SpecifierSet(requires_python):
continue
except InvalidSpecifier:
logging.warning(
"Invalid python_requires %r for package %s",
requires_python,
package_name,
)
continue

version = self._version_from_link(href)
if version is not None:
# Several compatible wheels can represent the same release.
compatible_versions.add(version)

return sorted(compatible_versions)

def _head_ok(self, url: str) -> bool:
"""Return True if a HEAD request to *url* resolves successfully (2xx)."""
headers = self._auth_header()
Expand Down
65 changes: 32 additions & 33 deletions eng/tools/azure-sdk-tools/pypi_tools/pypi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,70 +45,69 @@ def __init__(self, host="https://pypi.org", force_pypi=False):
ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None),
)

def _pypi_http(self):
"""Lazy PoolManager for pypi.org fallback when on AzDO backend."""
if not hasattr(self, "_pypi_http_pool"):
self._pypi_http_pool = PoolManager(
retries=Retry(total=3, raise_on_status=True),
ca_certs=os.getenv("REQUESTS_CA_BUNDLE", None),
)
return self._pypi_http_pool

def _pypi_json_request(self, path):
"""GET from pypi.org JSON API, using the active backend's http pool if on pypi, else fallback."""
if self._backend == "pypi":
url = "{host}{path}".format(host=self._host, path=path)
response = self._http.request("get", url)
else:
url = "https://pypi.org{path}".format(path=path)
response = self._pypi_http().request("get", url)
"""GET from the configured public PyPI JSON endpoint."""
if self._backend != "pypi":
raise NotImplementedError("PyPI JSON requests are unavailable against Azure Artifacts")
url = "{host}{path}".format(host=self._host, path=path)
response = self._http.request("get", url)
return json.loads(response.data.decode("utf-8"))

# ------------------------------------------------------------------
# PyPI JSON endpoints (fall back to pypi.org when on AzDO backend)
# PyPI JSON endpoints
# ------------------------------------------------------------------

def project(self, package_name):
if self._backend != "pypi":
raise NotImplementedError("project() is only available against pypi.org")
return self._pypi_json_request("/pypi/{}/json".format(package_name))

def project_release(self, package_name, version):
return self._pypi_json_request("/pypi/{}/{}/json".format(package_name, version))

# ------------------------------------------------------------------
# Shared interface
# ------------------------------------------------------------------

def filter_packages_for_compatibility(self, package_name, version_set):
def filter_packages_for_compatibility(self, package_name, version_set, project=None):
if self._backend != "pypi":
raise NotImplementedError(
"filter_packages_for_compatibility() requires pypi.org (needs requires_python metadata)"
)
from packaging.specifiers import InvalidSpecifier, SpecifierSet

project = project or self.project(package_name)
releases = project.get("releases", {})
current_python = parse(".".join(map(str, sys.version_info[:3])))
results: List[Version] = []
for version in version_set:
requires_python = self.project_release(package_name, version)["info"]["requires_python"]
if requires_python:
files = releases.get(str(version), [])
compatible = False
for release_file in files:
if release_file.get("yanked", False):
continue
requires_python = release_file.get("requires_python")
if not requires_python:
compatible = True
break
try:
if parse(".".join(map(str, sys.version_info[:3]))) in SpecifierSet(requires_python):
results.append(version)
if current_python in SpecifierSet(requires_python):
compatible = True
break
except InvalidSpecifier:
logging.warn(f"Invalid python_requires {requires_python!r} for package {package_name}=={version}")
logging.warning(
"Invalid python_requires %r for package %s==%s",
requires_python,
package_name,
version,
)
continue
else:
if compatible:
results.append(version)
return results

def get_ordered_versions(self, package_name, filter_by_compatibility=False) -> List[Version]:
if self._backend == "azdo":
versions = self._azdo.get_ordered_versions(package_name)
if filter_by_compatibility:
logging.warning(
"filter_by_compatibility is not supported against Azure Artifacts; returning unfiltered versions"
)
return versions
return self._azdo.get_python_compatible_versions(package_name)
return self._azdo.get_ordered_versions(package_name)

project = self.project(package_name)
versions: List[Version] = []
Expand All @@ -123,7 +122,7 @@ def get_ordered_versions(self, package_name, filter_by_compatibility=False) -> L
versions.sort()

if filter_by_compatibility:
return self.filter_packages_for_compatibility(package_name, versions)
return self.filter_packages_for_compatibility(package_name, versions, project)

return versions

Expand Down
Loading
Loading