diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 25bc61137066..6fff601f2878 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -22,6 +22,13 @@ parameters: - name: EnableCompiledCodeql type: boolean default: false +# Comma-separated 1ES network isolation flags. Defaults to enforcing CFSClean, which +# restricts outbound package restores to Microsoft CFS / Azure Artifacts endpoints. +# Pipelines that genuinely cannot restore from CFS (see conda-sdk-client.yml) may +# lower this to 'Permissive'. +- name: NetworkIsolationPolicy + type: string + default: Permissive, CFSClean extends: @@ -40,7 +47,7 @@ extends: - 1ES.PT.Tag-refs/tags/canary settings: skipBuildTagsForGitHubPullRequests: true - networkIsolationPolicy: Permissive + networkIsolationPolicy: ${{ parameters.NetworkIsolationPolicy }} ${{ if ne(variables['Build.DefinitionName'], 'python - core') }}: featureFlags: autoBaseline: false diff --git a/eng/pipelines/templates/stages/archetype-conda-release.yml b/eng/pipelines/templates/stages/archetype-conda-release.yml index 9e1595db8d78..62008c98ab46 100644 --- a/eng/pipelines/templates/stages/archetype-conda-release.yml +++ b/eng/pipelines/templates/stages/archetype-conda-release.yml @@ -34,10 +34,16 @@ stages: runOnce: deploy: steps: - - task: UsePythonVersion@0 - inputs: + - template: /eng/pipelines/templates/steps/use-python-version.yml + parameters: versionSpec: '3.12' + # Authenticate to the Azure Artifacts feed before any pip install. + # Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org. + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + EnableTwineAuth: false + - pwsh: | Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}} -Filter "*.conda" workingDirectory: $(Pipeline.Workspace) diff --git a/eng/pipelines/templates/stages/conda-sdk-client.yml b/eng/pipelines/templates/stages/conda-sdk-client.yml index 659c5a4d764b..c8e1beeb6545 100644 --- a/eng/pipelines/templates/stages/conda-sdk-client.yml +++ b/eng/pipelines/templates/stages/conda-sdk-client.yml @@ -220,6 +220,13 @@ parameters: extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: + # conda cannot restore from CFS. `conda env create` resolves against + # conda.anaconda.org / repo.anaconda.com, conda has no PIP_INDEX_URL equivalent, + # and Azure Artifacts has no conda feed type, so CFSClean severs the connection + # before the build environment can even be created. Everything pip-based in this + # pipeline does restore from the CFS feed. Revisit if a mirrored conda channel or + # a pre-baked pool image becomes available. + NetworkIsolationPolicy: Permissive stages: - stage: Build_Universal_Dependencies displayName: Build Universal Conda Packages diff --git a/eng/pipelines/templates/steps/build-conda-artifacts.yml b/eng/pipelines/templates/steps/build-conda-artifacts.yml index 57a0bb875375..2c142662ad1f 100644 --- a/eng/pipelines/templates/steps/build-conda-artifacts.yml +++ b/eng/pipelines/templates/steps/build-conda-artifacts.yml @@ -8,13 +8,24 @@ parameters: - name: Arguments type: string default: '' + - name: DevFeedName + type: string + default: 'public/azure-sdk-for-python' steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - inputs: + # Sets a default PIP_INDEX_URL before UsePythonVersion@0 so the task's own pip + # auto-restore does not reach pypi.org under network isolation. + - template: /eng/pipelines/templates/steps/use-python-version.yml + parameters: versionSpec: $(PythonVersion) + # Authenticate to the Azure Artifacts feed before any pip install. + # Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org. + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + - pwsh: | $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true diff --git a/eng/scripts/seed-virtualenv-wheels.ps1 b/eng/scripts/seed-virtualenv-wheels.ps1 index 9215588e84d4..7935e735803a 100644 --- a/eng/scripts/seed-virtualenv-wheels.ps1 +++ b/eng/scripts/seed-virtualenv-wheels.ps1 @@ -27,8 +27,14 @@ param ( $attempts = 0 -# ensure these can be pulled down from pypi. -$env:PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple" +# virtualenv --download shells out to pip, which reads PIP_INDEX_URL. Prefer whatever the pipeline +# already authenticated; only fall back to the public CFS feed when nothing is set. +if (-not $env:PIP_INDEX_URL) { + $env:PIP_INDEX_URL = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + Write-Host "PIP_INDEX_URL was not set; defaulting to the public azure-sdk-for-python feed." +} else { + Write-Host "PIP_INDEX_URL is already set; preserving existing value." +} while ($attempts -lt 3) { virtualenv --download --reset-app-data ` diff --git a/eng/scripts/set_checks.py b/eng/scripts/set_checks.py index fc6144e11d97..f65a75ef6c7b 100644 --- a/eng/scripts/set_checks.py +++ b/eng/scripts/set_checks.py @@ -17,10 +17,15 @@ "sdist", "import_all", "latestdependency", - "mindependency", + # Testing mindependency is disabled for CFS onboarding. + # https://github.com/Azure/azure-sdk-for-python/issues/48346 + # "mindependency", "whl_no_aio", ] -PR_BUILD_SET = ["whl", "sdist", "mindependency"] + +# Testing mindependency is disabled for CFS onboarding. +# https://github.com/Azure/azure-sdk-for-python/issues/48346 +PR_BUILD_SET = ["whl", "sdist"] #, "mindependency"] def resolve_devops_variable(var_value: str) -> List[str]: diff --git a/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py b/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py index 08901e9e8a1d..b7c078fdd689 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py +++ b/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py @@ -1,10 +1,7 @@ from typing import List, Any, Optional import os -import bs4 -import urllib3 from ci_tools.variables import str_to_bool -http = urllib3.PoolManager() # arguments: | # -c "${{ replace(convertToJson(parameters.CondaArtifacts), '"', '\"') }}" # -w "$(Build.SourcesDirectory)/conda/conda-recipes" @@ -48,22 +45,6 @@ # version: 12.7.0 -def get_package_sdist_url(package: str, version: str) -> str: - url = f"https://pypi.org/pypi/{package}/{version}/json" - response = http.request("GET", url) - - if response.status != 200: - raise RuntimeError(f"Failed to fetch metadata for {package}@{version} from PyPI.") - - data = response.json() - - for file_info in data.get("urls", []): - if file_info.get("packagetype") == "sdist": - return file_info["url"] - - raise ValueError(f"Unable to find a source distribution for {package}@{version}.") - - class CheckoutConfiguration: def __init__(self, raw_json: dict): # we should always have a package name @@ -77,10 +58,14 @@ def __init__(self, raw_json: dict): self.version = raw_json.get("version", None) self.download_uri = raw_json.get("download_uri", None) - if self.version and self.checkout_path is None: - self.download_uri = get_package_sdist_url(self.package, self.version) + # A package identified only by name + version is sourced from a package index rather than + # from a git checkout. Resolution is deferred to download time so that it can be performed + # by pip against PIP_INDEX_URL, rather than by a direct call to the public PyPI API here. + # Resolving eagerly would also force a network call for every configured package, including + # ones that are not part of the current batch. + self.from_package_index = bool(self.version and self.checkout_path is None) - if not self.checkout_path and not self.download_uri: + if not self.checkout_path and not self.download_uri and not self.from_package_index: raise ValueError( "When defining a checkout configuration, one must either have a valid PyPI download url" " (download_uri) or a path and version in the repo (checkout_path, version)." @@ -90,6 +75,8 @@ def __str__(self) -> str: if self.download_uri: return f"""- {self.package} downloaded from pypi {self.download_uri}""" + elif self.from_package_index: + return f"- {self.package}=={self.version} downloaded from the configured package index" else: return f"""- {self.checkout_path}/{self.package} from git @ {self.version}""" diff --git a/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py b/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py index e42c8b0c350c..8cbbf5838253 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py +++ b/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py @@ -17,6 +17,8 @@ import json import shlex import subprocess +import sys +import tempfile import urllib3 from shutil import rmtree @@ -267,7 +269,7 @@ def create_combined_sdist( environment_config, ) - if conda_build.checkout[0].download_uri: + if conda_build.checkout[0].download_uri or conda_build.checkout[0].from_package_index: # if we have a single dependency that is downloadable, it will be placed in final sdist location # by the get_package_source function. In that case, we just need to find it and return it if singular_dependency: @@ -378,6 +380,60 @@ def download_pypi_source(target_folder: str, target_uri: str) -> str: return file_name +def download_sdist_from_index(target_folder: str, package: str, version: str) -> str: + """ + Downloads the source distribution for a package from the configured package index. + + pip is used rather than a direct HTTP call so that PIP_INDEX_URL is honored. Under network + isolation that variable points at an authenticated Azure Artifacts (CFS) feed, and public + package hosts are unreachable. + """ + os.makedirs(target_folder, exist_ok=True) + + with tempfile.TemporaryDirectory() as download_staging: + check_call( + [ + sys.executable, + "-m", + "pip", + "download", + f"{package}=={version}", + "--no-deps", + "--no-binary", + ":all:", + "--dest", + download_staging, + ] + ) + + downloaded = [f for f in os.listdir(download_staging) if os.path.isfile(os.path.join(download_staging, f))] + + if not downloaded: + raise RuntimeError(f"pip did not produce a source distribution for {package}=={version}.") + + if len(downloaded) > 1: + raise RuntimeError( + f"Expected exactly one source distribution for {package}=={version}, got: {sorted(downloaded)}." + ) + + file_name = os.path.join(target_folder, downloaded[0]) + + if not os.path.exists(file_name): + shutil.move(os.path.join(download_staging, downloaded[0]), file_name) + + return file_name + + +def resolve_package_source(checkout_config: CheckoutConfiguration, target_folder: str) -> str: + """ + Places the source distribution for a checkout configuration into target_folder and returns its path. + """ + if checkout_config.download_uri: + return download_pypi_source(target_folder, checkout_config.download_uri) + + return download_sdist_from_index(target_folder, checkout_config.package, checkout_config.version) + + def get_package_source( checkout_config: CheckoutConfiguration, download_folder: str, @@ -392,10 +448,10 @@ def get_package_source( # if we have a single package, we can simply use the source distribution _as is_ rather than # repackaging it. so we download and move it directly to assembled if len(conda_build.checkout) == 1: - return download_pypi_source(output_folder, checkout_config.download_uri) + return resolve_package_source(checkout_config, output_folder) # in case of multiple external packages, we need to unzip the code into the same format as we do for a git clone else: - downloaded_zip = download_pypi_source(download_folder, checkout_config.download_uri) + downloaded_zip = resolve_package_source(checkout_config, download_folder) unzip_staging_folder = prep_directory(os.path.join(download_folder, checkout_config.package)) unzipped_staged = unzip_file_to_directory(downloaded_zip, unzip_staging_folder) assembly_location = prep_directory( diff --git a/eng/tools/azure-sdk-tools/ci_tools/variables.py b/eng/tools/azure-sdk-tools/ci_tools/variables.py index 5732d3deb9f0..0b9d6a2f8699 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/variables.py +++ b/eng/tools/azure-sdk-tools/ci_tools/variables.py @@ -102,7 +102,10 @@ def in_analyze_weekly() -> int: "VIRTUALENV_WHEEL": "0.45.1", "VIRTUALENV_PIP": "24.0", "VIRTUALENV_SETUPTOOLS": "75.3.2", - "PIP_EXTRA_INDEX_URL": "https://pypi.python.org/simple", + # Intentionally no PIP_EXTRA_INDEX_URL default. azpysdk.main already points PIP_INDEX_URL and + # UV_DEFAULT_INDEX at CFS_INDEX_URL when they are unset, and PipAuthenticate@1 supplies an + # authenticated value in CI. Adding an extra index here would duplicate that feed and would + # leak it into `--pypi` runs, which are meant to resolve from PyPI only. # I haven't spent much time looking to see if a variable exists when invoking uv run. there might be one already that we can depend # on for get_pip_command adjustment. "IN_UV": "1",