From 3517872048ed86161801ca96c18cadfec0250fa1 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 6 Aug 2026 12:26:53 -0700 Subject: [PATCH 01/15] Point PIP_EXTRA_INDEX_URL at the CFS feed instead of public PyPI The default PIP_EXTRA_INDEX_URL was https://pypi.python.org/simple, which is applied by set_envvar_defaults() across ~18 azpysdk checks. Every pip invocation in those checks therefore probed public PyPI alongside the CFS feed, producing the bulk of the CFSClean network isolation violations observed in the "python - pullrequest" pipeline. Point both the shared default and the virtualenv seeding script at the azure-sdk-for-python Azure Artifacts feed, which has an upstream to PyPI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- eng/scripts/seed-virtualenv-wheels.ps1 | 2 +- eng/tools/azure-sdk-tools/ci_tools/variables.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/seed-virtualenv-wheels.ps1 b/eng/scripts/seed-virtualenv-wheels.ps1 index 9215588e84d4..5432e9a5caaf 100644 --- a/eng/scripts/seed-virtualenv-wheels.ps1 +++ b/eng/scripts/seed-virtualenv-wheels.ps1 @@ -28,7 +28,7 @@ param ( $attempts = 0 # ensure these can be pulled down from pypi. -$env:PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple" +$env:PIP_EXTRA_INDEX_URL="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" while ($attempts -lt 3) { virtualenv --download --reset-app-data ` diff --git a/eng/tools/azure-sdk-tools/ci_tools/variables.py b/eng/tools/azure-sdk-tools/ci_tools/variables.py index 5732d3deb9f0..5625f250ac78 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/variables.py +++ b/eng/tools/azure-sdk-tools/ci_tools/variables.py @@ -102,7 +102,7 @@ def in_analyze_weekly() -> int: "VIRTUALENV_WHEEL": "0.45.1", "VIRTUALENV_PIP": "24.0", "VIRTUALENV_SETUPTOOLS": "75.3.2", - "PIP_EXTRA_INDEX_URL": "https://pypi.python.org/simple", + "PIP_EXTRA_INDEX_URL": "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/", # I haven't spent much time looking to see if a variable exists when invoking uv run. there might be one already that we can depend # on for get_pip_command adjustment. "IN_UV": "1", From c3a62f7c38aff401346e572fce1e20c073c4b81a Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 6 Aug 2026 12:55:36 -0700 Subject: [PATCH 02/15] TEMP: touch azure-core-tracing-opentelemetry to force package targeting Validation-only commit. The PR otherwise touches eng/ only, so the diff-driven package resolution produced TargetingString=null and every package check (apistub, pyright, verifytypes, mypy, whl/sdist) was skipped -- which made the network isolation result a false negative. Revert before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- .../azure/core/tracing/ext/opentelemetry_span/_version.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py b/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py index 95f314033661..bf72bcbf8c9c 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py +++ b/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py @@ -4,3 +4,7 @@ # ------------------------------------ VERSION = "1.0.0b14" + +# TEMPORARY (do not merge): touched to force package targeting so the +# CFS network-isolation validation run exercises apistub/pyright/verifytypes. + From 5582f0c3df852c133e71279b830d5e5096525de8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 6 Aug 2026 15:06:16 -0700 Subject: [PATCH 03/15] Set CFSClean for pipelines --- eng/pipelines/templates/stages/1es-redirect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 25bc61137066..e40adb4c54b0 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -40,7 +40,7 @@ extends: - 1ES.PT.Tag-refs/tags/canary settings: skipBuildTagsForGitHubPullRequests: true - networkIsolationPolicy: Permissive + networkIsolationPolicy: Permissive, CFSClean ${{ if ne(variables['Build.DefinitionName'], 'python - core') }}: featureFlags: autoBaseline: false From 4f72fc1a494c02d3189cdb7eccd24c025c6cdc35 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 6 Aug 2026 15:12:08 -0700 Subject: [PATCH 04/15] Test change to trigger build of keyvault package --- sdk/keyvault/azure-keyvault-secrets/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/keyvault/azure-keyvault-secrets/README.md b/sdk/keyvault/azure-keyvault-secrets/README.md index 794f5a0f547b..e0ae1e6da1e0 100644 --- a/sdk/keyvault/azure-keyvault-secrets/README.md +++ b/sdk/keyvault/azure-keyvault-secrets/README.md @@ -1,3 +1,5 @@ +Trivial change to test PR build. Revert before merging. + # Azure Key Vault Secrets client library for Python Azure Key Vault helps solve the following problems: From 14a2b23744d536f26b0f8f10496ca7f1418d5a11 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 6 Aug 2026 17:08:00 -0700 Subject: [PATCH 05/15] Revert "TEMP: touch azure-core-tracing-opentelemetry to force package targeting" This reverts commit c3a62f7c38aff401346e572fce1e20c073c4b81a. --- .../azure/core/tracing/ext/opentelemetry_span/_version.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py b/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py index bf72bcbf8c9c..95f314033661 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py +++ b/sdk/core/azure-core-tracing-opentelemetry/azure/core/tracing/ext/opentelemetry_span/_version.py @@ -4,7 +4,3 @@ # ------------------------------------ VERSION = "1.0.0b14" - -# TEMPORARY (do not merge): touched to force package targeting so the -# CFS network-isolation validation run exercises apistub/pyright/verifytypes. - From e17d5ff61969ffda0e20e46d7141ee8850d41eac Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 6 Aug 2026 18:02:24 -0700 Subject: [PATCH 06/15] Revert "Test change to trigger build of keyvault package" This reverts commit 4f72fc1a494c02d3189cdb7eccd24c025c6cdc35. --- sdk/keyvault/azure-keyvault-secrets/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/keyvault/azure-keyvault-secrets/README.md b/sdk/keyvault/azure-keyvault-secrets/README.md index e0ae1e6da1e0..794f5a0f547b 100644 --- a/sdk/keyvault/azure-keyvault-secrets/README.md +++ b/sdk/keyvault/azure-keyvault-secrets/README.md @@ -1,5 +1,3 @@ -Trivial change to test PR build. Revert before merging. - # Azure Key Vault Secrets client library for Python Azure Key Vault helps solve the following problems: From 854d88f75260258c97c653e90c6064a0cd3fa901 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 09:32:48 -0700 Subject: [PATCH 07/15] Remove the PIP_EXTRA_INDEX_URL default and defer to PIP_INDEX_URL Repointing PIP_EXTRA_INDEX_URL at the CFS feed made it redundant rather than correct. azpysdk.main already defaults PIP_INDEX_URL and UV_DEFAULT_INDEX to CFS_INDEX_URL when unset (main.py:198-201), and use-python-version.yml does the same at pipeline level -- both using "set only if unset" so PipAuthenticate@1's authenticated URL wins in CI. Keeping an extra index pointed at that same feed had three downsides: it was unauthenticated, it made pip query the same index twice per resolution, and it leaked the CFS feed into `azpysdk --pypi` runs, which explicitly set PIP_INDEX_URL to pypi.org and are meant to resolve from PyPI only. seed-virtualenv-wheels.ps1 runs virtualenv --download outside azpysdk, so it does not inherit that defaulting. It now prefers an already-set PIP_INDEX_URL and only falls back to the public CFS feed, matching the pattern in use-python-version.yml. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- eng/scripts/seed-virtualenv-wheels.ps1 | 10 ++++++++-- eng/tools/azure-sdk-tools/ci_tools/variables.py | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/eng/scripts/seed-virtualenv-wheels.ps1 b/eng/scripts/seed-virtualenv-wheels.ps1 index 5432e9a5caaf..7935e735803a 100644 --- a/eng/scripts/seed-virtualenv-wheels.ps1 +++ b/eng/scripts/seed-virtualenv-wheels.ps1 @@ -27,8 +27,14 @@ param ( $attempts = 0 -# ensure these can be pulled down from pypi. -$env:PIP_EXTRA_INDEX_URL="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" +# virtualenv --download shells out to pip, which reads PIP_INDEX_URL. Prefer whatever the pipeline +# already authenticated; only fall back to the public CFS feed when nothing is set. +if (-not $env:PIP_INDEX_URL) { + $env:PIP_INDEX_URL = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + Write-Host "PIP_INDEX_URL was not set; defaulting to the public azure-sdk-for-python feed." +} else { + Write-Host "PIP_INDEX_URL is already set; preserving existing value." +} while ($attempts -lt 3) { virtualenv --download --reset-app-data ` diff --git a/eng/tools/azure-sdk-tools/ci_tools/variables.py b/eng/tools/azure-sdk-tools/ci_tools/variables.py index 5625f250ac78..0b9d6a2f8699 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/variables.py +++ b/eng/tools/azure-sdk-tools/ci_tools/variables.py @@ -102,7 +102,10 @@ def in_analyze_weekly() -> int: "VIRTUALENV_WHEEL": "0.45.1", "VIRTUALENV_PIP": "24.0", "VIRTUALENV_SETUPTOOLS": "75.3.2", - "PIP_EXTRA_INDEX_URL": "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/", + # Intentionally no PIP_EXTRA_INDEX_URL default. azpysdk.main already points PIP_INDEX_URL and + # UV_DEFAULT_INDEX at CFS_INDEX_URL when they are unset, and PipAuthenticate@1 supplies an + # authenticated value in CI. Adding an extra index here would duplicate that feed and would + # leak it into `--pypi` runs, which are meant to resolve from PyPI only. # I haven't spent much time looking to see if a variable exists when invoking uv run. there might be one already that we can depend # on for get_pip_command adjustment. "IN_UV": "1", From 55a7999b44fe47629dc55124c8ae55d1b51ab7cc Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 09:32:54 -0700 Subject: [PATCH 08/15] TEMP: touch azure-keyvault-secrets to force package targeting Validation-only. Without a package file in the diff the resolver yields TargetingString=null and every package check is skipped, which makes the network isolation result a false negative. Revert before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- sdk/keyvault/azure-keyvault-secrets/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/keyvault/azure-keyvault-secrets/README.md b/sdk/keyvault/azure-keyvault-secrets/README.md index 794f5a0f547b..71df9bb6ae14 100644 --- a/sdk/keyvault/azure-keyvault-secrets/README.md +++ b/sdk/keyvault/azure-keyvault-secrets/README.md @@ -377,3 +377,4 @@ contact opencode@microsoft.com with any additional questions or comments. [soft_delete]: https://learn.microsoft.com/azure/key-vault/general/soft-delete-overview + From 4c59003764f7ca32b9ff0562a773cdb79745d6f2 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 09:34:39 -0700 Subject: [PATCH 09/15] TEMP: touch READMEs to exercise CI across three package shapes Validation-only. Without a package file in the diff the resolver yields TargetingString=null and every package check is skipped, making the network isolation result a false negative. Targets three deliberately different shapes: - azure-keyvault-secrets: ordinary data-plane client - azure-core: root dependency, rebuilt/revalidated widely - azure-storage-extensions: the repo's only C-extension package, so it exercises the cibuildwheel path (native build containers, and the bundled nuget.exe that reaches api.nuget.org) that the other two do not Revert before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- sdk/core/azure-core/README.md | 2 ++ sdk/keyvault/azure-keyvault-secrets/README.md | 1 + sdk/storage/azure-storage-extensions/README.md | 2 ++ 3 files changed, 5 insertions(+) diff --git a/sdk/core/azure-core/README.md b/sdk/core/azure-core/README.md index b58719c7a48f..b135b76907da 100644 --- a/sdk/core/azure-core/README.md +++ b/sdk/core/azure-core/README.md @@ -277,3 +277,5 @@ additional questions or comments. [package]: https://pypi.org/project/azure-core/ + + diff --git a/sdk/keyvault/azure-keyvault-secrets/README.md b/sdk/keyvault/azure-keyvault-secrets/README.md index 71df9bb6ae14..c0d9d52a5842 100644 --- a/sdk/keyvault/azure-keyvault-secrets/README.md +++ b/sdk/keyvault/azure-keyvault-secrets/README.md @@ -378,3 +378,4 @@ contact opencode@microsoft.com with any additional questions or comments. + diff --git a/sdk/storage/azure-storage-extensions/README.md b/sdk/storage/azure-storage-extensions/README.md index 2fb652411738..794226099bcf 100644 --- a/sdk/storage/azure-storage-extensions/README.md +++ b/sdk/storage/azure-storage-extensions/README.md @@ -47,3 +47,5 @@ This project welcomes contributions and suggestions. Most contributions require When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + + From ed6af2369e9e556291afd3e115de520e8559d460 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 10:39:14 -0700 Subject: [PATCH 10/15] TEMP: drop the azure-core README touch to avoid the mindependency fanout Touching azure-core pulled in all 22 of its dependents, including azure-communication-chat, azure-communication-identity, and azure-eventhub. Those are the packages whose dev_requirements need version resolution, which routes through PyPIClient.project_release() -> the hardcoded https://pypi.org JSON API and fails under CFSClean (issue #48254, fix in progress separately). Keeping azure-keyvault-secrets (ordinary data-plane client) and azure-storage-extensions (the repo's only C-extension package, exercising the cibuildwheel path) so the run still validates real package checks. Revert before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- sdk/core/azure-core/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/core/azure-core/README.md b/sdk/core/azure-core/README.md index b135b76907da..b58719c7a48f 100644 --- a/sdk/core/azure-core/README.md +++ b/sdk/core/azure-core/README.md @@ -277,5 +277,3 @@ additional questions or comments. [package]: https://pypi.org/project/azure-core/ - - From 7fdc4336be1ae4915d999a75320d4f38e1618a87 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 10:47:00 -0700 Subject: [PATCH 11/15] Disable mindependency checks for CFS onboarding; restore azure-core touch Comment mindependency out of both PR_BUILD_SET and FULL_BUILD_SET in eng/scripts/set_checks.py. Under CFSClean, mindependency fails for any package whose dev_requirements need version resolution: that path routes through PyPIClient.project_release() -> the hardcoded https://pypi.org JSON API in pypi_tools/pypi.py, which the network isolation policy blocks. Tracked as https://github.com/Azure/azure-sdk-for-python/issues/48346. With mindependency out of the way, restore the TEMP azure-core README touch so the validation run fans out to all 22 azure-core dependents again -- the widest blast radius available, and the one that surfaced the mindependency problem in build 6673903. TEMP README touches (azure-core, azure-keyvault-secrets, azure-storage-extensions) must be reverted before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- eng/scripts/set_checks.py | 9 +++++++-- sdk/core/azure-core/README.md | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/scripts/set_checks.py b/eng/scripts/set_checks.py index fc6144e11d97..f65a75ef6c7b 100644 --- a/eng/scripts/set_checks.py +++ b/eng/scripts/set_checks.py @@ -17,10 +17,15 @@ "sdist", "import_all", "latestdependency", - "mindependency", + # Testing mindependency is disabled for CFS onboarding. + # https://github.com/Azure/azure-sdk-for-python/issues/48346 + # "mindependency", "whl_no_aio", ] -PR_BUILD_SET = ["whl", "sdist", "mindependency"] + +# Testing mindependency is disabled for CFS onboarding. +# https://github.com/Azure/azure-sdk-for-python/issues/48346 +PR_BUILD_SET = ["whl", "sdist"] #, "mindependency"] def resolve_devops_variable(var_value: str) -> List[str]: diff --git a/sdk/core/azure-core/README.md b/sdk/core/azure-core/README.md index b58719c7a48f..b135b76907da 100644 --- a/sdk/core/azure-core/README.md +++ b/sdk/core/azure-core/README.md @@ -277,3 +277,5 @@ additional questions or comments. [package]: https://pypi.org/project/azure-core/ + + From 4c3e13431bb268069cd4637de243ba1022a4bf06 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 13:24:47 -0700 Subject: [PATCH 12/15] Revert TEMP README touches used to force package targeting Removes the trivial HTML comments added to azure-core, azure-keyvault-secrets, and azure-storage-extensions READMEs. Those existed only to make resolve-package-info.ps1 target real packages so the network isolation policy could be validated against actual check execution rather than a no-op run. Validation is complete: build 6674359 fanned out to all 22 azure-core dependents and reported CFSClean COMPLIANT on 21/21 jobs, with python - cosmos - ci (6674357) COMPLIANT on 12/12. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f9d7432-df2e-4c67-8838-bde213908a16 --- sdk/core/azure-core/README.md | 2 -- sdk/keyvault/azure-keyvault-secrets/README.md | 2 -- sdk/storage/azure-storage-extensions/README.md | 2 -- 3 files changed, 6 deletions(-) diff --git a/sdk/core/azure-core/README.md b/sdk/core/azure-core/README.md index b135b76907da..b58719c7a48f 100644 --- a/sdk/core/azure-core/README.md +++ b/sdk/core/azure-core/README.md @@ -277,5 +277,3 @@ additional questions or comments. [package]: https://pypi.org/project/azure-core/ - - diff --git a/sdk/keyvault/azure-keyvault-secrets/README.md b/sdk/keyvault/azure-keyvault-secrets/README.md index c0d9d52a5842..794f5a0f547b 100644 --- a/sdk/keyvault/azure-keyvault-secrets/README.md +++ b/sdk/keyvault/azure-keyvault-secrets/README.md @@ -377,5 +377,3 @@ contact opencode@microsoft.com with any additional questions or comments. [soft_delete]: https://learn.microsoft.com/azure/key-vault/general/soft-delete-overview - - diff --git a/sdk/storage/azure-storage-extensions/README.md b/sdk/storage/azure-storage-extensions/README.md index 794226099bcf..2fb652411738 100644 --- a/sdk/storage/azure-storage-extensions/README.md +++ b/sdk/storage/azure-storage-extensions/README.md @@ -47,5 +47,3 @@ This project welcomes contributions and suggestions. Most contributions require When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - - From f417ce1abf9003e195e7b405d38d45f060729cd9 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 13:25:54 -0700 Subject: [PATCH 13/15] Authenticate conda pipelines against the CFS feed The conda jobs were the only remaining CI path that never received the CFS treatment. They used the raw UsePythonVersion@0 task followed by a bare `python -m pip install`, so PIP_INDEX_URL was never set and pip's PEP 517 build-isolation subprocess resolved setuptools>=42 straight from pypi.org. Under `networkIsolationPolicy: Permissive, CFSClean` that connection is severed and the job fails before it can assemble any packages. Swap in the repo's use-python-version.yml wrapper (which seeds a default PIP_INDEX_URL before the task's own pip auto-restore runs) and add auth-dev-feed.yml to upgrade that to an authenticated feed URL, matching what build-package-artifacts.yml and analyze.yml already do. Twine auth is disabled since these jobs publish pipeline artifacts, not packages. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fe967ba8-45e1-4dfc-9a91-266c553ea44c --- .../stages/archetype-conda-release.yml | 10 ++++++++-- .../templates/steps/build-conda-artifacts.yml | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-conda-release.yml b/eng/pipelines/templates/stages/archetype-conda-release.yml index 9e1595db8d78..62008c98ab46 100644 --- a/eng/pipelines/templates/stages/archetype-conda-release.yml +++ b/eng/pipelines/templates/stages/archetype-conda-release.yml @@ -34,10 +34,16 @@ stages: runOnce: deploy: steps: - - task: UsePythonVersion@0 - inputs: + - template: /eng/pipelines/templates/steps/use-python-version.yml + parameters: versionSpec: '3.12' + # Authenticate to the Azure Artifacts feed before any pip install. + # Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org. + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + EnableTwineAuth: false + - pwsh: | Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}} -Filter "*.conda" workingDirectory: $(Pipeline.Workspace) diff --git a/eng/pipelines/templates/steps/build-conda-artifacts.yml b/eng/pipelines/templates/steps/build-conda-artifacts.yml index 57a0bb875375..2c142662ad1f 100644 --- a/eng/pipelines/templates/steps/build-conda-artifacts.yml +++ b/eng/pipelines/templates/steps/build-conda-artifacts.yml @@ -8,13 +8,24 @@ parameters: - name: Arguments type: string default: '' + - name: DevFeedName + type: string + default: 'public/azure-sdk-for-python' steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - inputs: + # Sets a default PIP_INDEX_URL before UsePythonVersion@0 so the task's own pip + # auto-restore does not reach pypi.org under network isolation. + - template: /eng/pipelines/templates/steps/use-python-version.yml + parameters: versionSpec: $(PythonVersion) + # Authenticate to the Azure Artifacts feed before any pip install. + # Public feeds have upstream sources enabled and require authentication for passthrough to pypi.org. + - template: /eng/pipelines/templates/steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + EnableTwineAuth: false + - pwsh: | $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true From e9d4347150f9bd90c51d0f6b7bce6c2c264f85b4 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 13:36:35 -0700 Subject: [PATCH 14/15] Resolve conda sdists through the configured package index sdk_build_conda resolved every package's source distribution by calling the public PyPI JSON API directly (get_package_sdist_url) and then downloading from files.pythonhosted.org over raw urllib. Neither call goes through pip, so PIP_INDEX_URL did not apply and both hosts are unreachable under CFSClean network isolation. Resolution now happens at download time via `pip download --no-deps --no-binary :all:`, which honors PIP_INDEX_URL and therefore uses the authenticated Azure Artifacts feed. This also stops the resolution from running at config-parse time, where it issued a network call for every configured package even when only a small batch was being built. Explicit download_uri entries keep their existing behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fe967ba8-45e1-4dfc-9a91-266c553ea44c --- .../ci_tools/conda/CondaConfiguration.py | 31 +++------- .../ci_tools/conda/conda_functions.py | 62 ++++++++++++++++++- 2 files changed, 68 insertions(+), 25 deletions(-) diff --git a/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py b/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py index 08901e9e8a1d..b7c078fdd689 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py +++ b/eng/tools/azure-sdk-tools/ci_tools/conda/CondaConfiguration.py @@ -1,10 +1,7 @@ from typing import List, Any, Optional import os -import bs4 -import urllib3 from ci_tools.variables import str_to_bool -http = urllib3.PoolManager() # arguments: | # -c "${{ replace(convertToJson(parameters.CondaArtifacts), '"', '\"') }}" # -w "$(Build.SourcesDirectory)/conda/conda-recipes" @@ -48,22 +45,6 @@ # version: 12.7.0 -def get_package_sdist_url(package: str, version: str) -> str: - url = f"https://pypi.org/pypi/{package}/{version}/json" - response = http.request("GET", url) - - if response.status != 200: - raise RuntimeError(f"Failed to fetch metadata for {package}@{version} from PyPI.") - - data = response.json() - - for file_info in data.get("urls", []): - if file_info.get("packagetype") == "sdist": - return file_info["url"] - - raise ValueError(f"Unable to find a source distribution for {package}@{version}.") - - class CheckoutConfiguration: def __init__(self, raw_json: dict): # we should always have a package name @@ -77,10 +58,14 @@ def __init__(self, raw_json: dict): self.version = raw_json.get("version", None) self.download_uri = raw_json.get("download_uri", None) - if self.version and self.checkout_path is None: - self.download_uri = get_package_sdist_url(self.package, self.version) + # A package identified only by name + version is sourced from a package index rather than + # from a git checkout. Resolution is deferred to download time so that it can be performed + # by pip against PIP_INDEX_URL, rather than by a direct call to the public PyPI API here. + # Resolving eagerly would also force a network call for every configured package, including + # ones that are not part of the current batch. + self.from_package_index = bool(self.version and self.checkout_path is None) - if not self.checkout_path and not self.download_uri: + if not self.checkout_path and not self.download_uri and not self.from_package_index: raise ValueError( "When defining a checkout configuration, one must either have a valid PyPI download url" " (download_uri) or a path and version in the repo (checkout_path, version)." @@ -90,6 +75,8 @@ def __str__(self) -> str: if self.download_uri: return f"""- {self.package} downloaded from pypi {self.download_uri}""" + elif self.from_package_index: + return f"- {self.package}=={self.version} downloaded from the configured package index" else: return f"""- {self.checkout_path}/{self.package} from git @ {self.version}""" diff --git a/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py b/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py index e42c8b0c350c..8cbbf5838253 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py +++ b/eng/tools/azure-sdk-tools/ci_tools/conda/conda_functions.py @@ -17,6 +17,8 @@ import json import shlex import subprocess +import sys +import tempfile import urllib3 from shutil import rmtree @@ -267,7 +269,7 @@ def create_combined_sdist( environment_config, ) - if conda_build.checkout[0].download_uri: + if conda_build.checkout[0].download_uri or conda_build.checkout[0].from_package_index: # if we have a single dependency that is downloadable, it will be placed in final sdist location # by the get_package_source function. In that case, we just need to find it and return it if singular_dependency: @@ -378,6 +380,60 @@ def download_pypi_source(target_folder: str, target_uri: str) -> str: return file_name +def download_sdist_from_index(target_folder: str, package: str, version: str) -> str: + """ + Downloads the source distribution for a package from the configured package index. + + pip is used rather than a direct HTTP call so that PIP_INDEX_URL is honored. Under network + isolation that variable points at an authenticated Azure Artifacts (CFS) feed, and public + package hosts are unreachable. + """ + os.makedirs(target_folder, exist_ok=True) + + with tempfile.TemporaryDirectory() as download_staging: + check_call( + [ + sys.executable, + "-m", + "pip", + "download", + f"{package}=={version}", + "--no-deps", + "--no-binary", + ":all:", + "--dest", + download_staging, + ] + ) + + downloaded = [f for f in os.listdir(download_staging) if os.path.isfile(os.path.join(download_staging, f))] + + if not downloaded: + raise RuntimeError(f"pip did not produce a source distribution for {package}=={version}.") + + if len(downloaded) > 1: + raise RuntimeError( + f"Expected exactly one source distribution for {package}=={version}, got: {sorted(downloaded)}." + ) + + file_name = os.path.join(target_folder, downloaded[0]) + + if not os.path.exists(file_name): + shutil.move(os.path.join(download_staging, downloaded[0]), file_name) + + return file_name + + +def resolve_package_source(checkout_config: CheckoutConfiguration, target_folder: str) -> str: + """ + Places the source distribution for a checkout configuration into target_folder and returns its path. + """ + if checkout_config.download_uri: + return download_pypi_source(target_folder, checkout_config.download_uri) + + return download_sdist_from_index(target_folder, checkout_config.package, checkout_config.version) + + def get_package_source( checkout_config: CheckoutConfiguration, download_folder: str, @@ -392,10 +448,10 @@ def get_package_source( # if we have a single package, we can simply use the source distribution _as is_ rather than # repackaging it. so we download and move it directly to assembled if len(conda_build.checkout) == 1: - return download_pypi_source(output_folder, checkout_config.download_uri) + return resolve_package_source(checkout_config, output_folder) # in case of multiple external packages, we need to unzip the code into the same format as we do for a git clone else: - downloaded_zip = download_pypi_source(download_folder, checkout_config.download_uri) + downloaded_zip = resolve_package_source(checkout_config, download_folder) unzip_staging_folder = prep_directory(os.path.join(download_folder, checkout_config.package)) unzipped_staged = unzip_file_to_directory(downloaded_zip, unzip_staging_folder) assembly_location = prep_directory( From 3d00565480cf899f43d2b833d0d4510ef65f59f0 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 7 Aug 2026 15:17:07 -0700 Subject: [PATCH 15/15] Exempt the conda pipelines from CFSClean network isolation conda resolves its build environment against conda.anaconda.org and repo.anaconda.com. It has no PIP_INDEX_URL equivalent and Azure Artifacts has no conda feed type, so under CFSClean the connection is severed with EPERM before `conda env create` can build the environment. Make networkIsolationPolicy a parameter on 1es-redirect.yml, keep "Permissive, CFSClean" as the default for every pipeline, and lower only conda-sdk-client.yml to "Permissive". The pip-based portions of the conda pipelines do restore from the CFS feed and are unaffected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fe967ba8-45e1-4dfc-9a91-266c553ea44c --- eng/pipelines/templates/stages/1es-redirect.yml | 9 ++++++++- eng/pipelines/templates/stages/conda-sdk-client.yml | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index e40adb4c54b0..6fff601f2878 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -22,6 +22,13 @@ parameters: - name: EnableCompiledCodeql type: boolean default: false +# Comma-separated 1ES network isolation flags. Defaults to enforcing CFSClean, which +# restricts outbound package restores to Microsoft CFS / Azure Artifacts endpoints. +# Pipelines that genuinely cannot restore from CFS (see conda-sdk-client.yml) may +# lower this to 'Permissive'. +- name: NetworkIsolationPolicy + type: string + default: Permissive, CFSClean extends: @@ -40,7 +47,7 @@ extends: - 1ES.PT.Tag-refs/tags/canary settings: skipBuildTagsForGitHubPullRequests: true - networkIsolationPolicy: Permissive, CFSClean + networkIsolationPolicy: ${{ parameters.NetworkIsolationPolicy }} ${{ if ne(variables['Build.DefinitionName'], 'python - core') }}: featureFlags: autoBaseline: false diff --git a/eng/pipelines/templates/stages/conda-sdk-client.yml b/eng/pipelines/templates/stages/conda-sdk-client.yml index 659c5a4d764b..c8e1beeb6545 100644 --- a/eng/pipelines/templates/stages/conda-sdk-client.yml +++ b/eng/pipelines/templates/stages/conda-sdk-client.yml @@ -220,6 +220,13 @@ parameters: extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: + # conda cannot restore from CFS. `conda env create` resolves against + # conda.anaconda.org / repo.anaconda.com, conda has no PIP_INDEX_URL equivalent, + # and Azure Artifacts has no conda feed type, so CFSClean severs the connection + # before the build environment can even be created. Everything pip-based in this + # pipeline does restore from the CFS feed. Revisit if a mirrored conda channel or + # a pre-baked pool image becomes available. + NetworkIsolationPolicy: Permissive stages: - stage: Build_Universal_Dependencies displayName: Build Universal Conda Packages