From ea16799f009747e7faac7b4142fe127caca4881f Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Tue, 31 Mar 2026 11:30:28 -0700 Subject: [PATCH 01/11] Add ADO pipeline hyperlinks to CI-AND-RELEASE-PIPELINES.md --- .Pipelines/CI-AND-RELEASE-PIPELINES.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.Pipelines/CI-AND-RELEASE-PIPELINES.md b/.Pipelines/CI-AND-RELEASE-PIPELINES.md index d8d18d00..ce0fd9b0 100644 --- a/.Pipelines/CI-AND-RELEASE-PIPELINES.md +++ b/.Pipelines/CI-AND-RELEASE-PIPELINES.md @@ -7,16 +7,16 @@ including what each pipeline does, when it runs, and how to trigger a release. ## Pipeline Files -| File | Purpose | -|------|---------| -| [`azure-pipelines.yml`](../azure-pipelines.yml) | PR gate and post-merge CI — calls the shared template with `runPublish: false` | -| [`pipeline-publish.yml`](pipeline-publish.yml) | Release pipeline — manually queued, builds and publishes to PyPI | -| [`template-pipeline-stages.yml`](template-pipeline-stages.yml) | Shared stages template — PreBuildCheck, Validate, and CI stages reused by both pipelines | -| [`credscan-exclusion.json`](credscan-exclusion.json) | CredScan suppression file for known test fixtures | +| File | ADO Pipeline | Purpose | +|------|-------------|---------| +| [`azure-pipelines.yml`](../azure-pipelines.yml) | [MSAL.Python-PR-OneBranch-Official (3064)](https://dev.azure.com/IdentityDivision/IDDP/_build?definitionId=3064) | PR gate and post-merge CI — calls the shared template with `runPublish: false` | +| [`pipeline-publish.yml`](pipeline-publish.yml) | [MSAL.Python-Publish (3067)](https://dev.azure.com/IdentityDivision/IDDP/_build?definitionId=3067) | Release pipeline — manually queued, builds and publishes to PyPI | +| [`template-pipeline-stages.yml`](template-pipeline-stages.yml) | — | Shared stages template — PreBuildCheck, Validate, and CI stages reused by both pipelines | +| [`credscan-exclusion.json`](credscan-exclusion.json) | — | CredScan suppression file for known test fixtures | --- -## PR / CI Pipeline (`azure-pipelines.yml`) +## PR / CI Pipeline — [MSAL.Python-PR-OneBranch-Official (3064)](https://dev.azure.com/IdentityDivision/IDDP/_build?definitionId=3064) ### Triggers @@ -45,7 +45,7 @@ The Validate stage is **skipped** on PR/CI runs (it only applies to release buil --- -## Release Pipeline (`pipeline-publish.yml`) +## Release Pipeline — [MSAL.Python-Publish (3067)](https://dev.azure.com/IdentityDivision/IDDP/_build?definitionId=3067) ### Triggers From 5eaf53356b40e355db777f3820bd85b3c550280e Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Tue, 31 Mar 2026 11:37:34 -0700 Subject: [PATCH 02/11] Add Python 3.8 to ADO CI matrix --- .Pipelines/CI-AND-RELEASE-PIPELINES.md | 4 ++-- .Pipelines/template-pipeline-stages.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.Pipelines/CI-AND-RELEASE-PIPELINES.md b/.Pipelines/CI-AND-RELEASE-PIPELINES.md index ce0fd9b0..e7c5ad1a 100644 --- a/.Pipelines/CI-AND-RELEASE-PIPELINES.md +++ b/.Pipelines/CI-AND-RELEASE-PIPELINES.md @@ -35,7 +35,7 @@ PreBuildCheck ─► CI | Stage | What it does | |-------|-------------| | **PreBuildCheck** | Runs SDL security scans: PoliCheck (policy/offensive content), CredScan (leaked credentials), and PostAnalysis (breaks the build on findings) | -| **CI** | Runs the full test suite on Python 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14 | +| **CI** | Runs the full test suite on Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14 | The Validate stage is **skipped** on PR/CI runs (it only applies to release builds). @@ -70,7 +70,7 @@ PreBuildCheck ─► Validate ─► CI ─► Build ─┬─► PublishMSALPyt |-------|-------------|-----------| | **PreBuildCheck** | PoliCheck + CredScan scans | Always | | **Validate** | Asserts the `packageVersion` parameter matches `msal/sku.py __version__` | Always (release runs only) | -| **CI** | Full test matrix (Python 3.9–3.14) | After Validate passes | +| **CI** | Full test matrix (Python 3.8–3.14) | After Validate passes | | **Build** | Builds `sdist` and `wheel` via `python -m build`; publishes `python-dist` artifact | After CI passes | | **PublishMSALPython** | Uploads to test.pypi.org | `publishTarget == test.pypi.org (Preview / RC)` | | **PublishPyPI** | Uploads to PyPI via ESRP; requires manual approval | `publishTarget == pypi.org (ESRP Production)` | diff --git a/.Pipelines/template-pipeline-stages.yml b/.Pipelines/template-pipeline-stages.yml index 972121c3..30984da7 100644 --- a/.Pipelines/template-pipeline-stages.yml +++ b/.Pipelines/template-pipeline-stages.yml @@ -127,6 +127,8 @@ stages: vmImage: ubuntu-latest strategy: matrix: + Python38: + python.version: '3.8' Python39: python.version: '3.9' Python310: From bd4163340a28bfac65a6fa81893af49b2409d67c Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Tue, 31 Mar 2026 11:53:46 -0700 Subject: [PATCH 03/11] Move benchmarks from GH Actions to ADO; remove cb job from GH workflow --- .github/workflows/python-package.yml | 82 ---------------------------- azure-pipelines.yml | 43 +++++++++++++++ 2 files changed, 43 insertions(+), 82 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6c1626b6..2fe09625 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,91 +5,9 @@ name: CI/CD on: push: - pull_request: - branches: [ dev ] - - # This guards against unknown PR until a community member vet it and label it. - types: [ labeled ] jobs: - ci: - env: - # Fake a TRAVIS env so that the pre-existing test cases would behave like before - TRAVIS: true - LAB_APP_CLIENT_ID: ${{ secrets.LAB_APP_CLIENT_ID }} - LAB_APP_CLIENT_CERT_BASE64: ${{ secrets.LAB_APP_CLIENT_CERT_BASE64 }} - LAB_APP_CLIENT_CERT_PFX_PATH: lab_cert.pfx - - # Derived from https://docs.github.com/en/actions/guides/building-and-testing-python#starting-with-the-python-workflow-template - runs-on: ubuntu-22.04 - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - # It automatically takes care of pip cache, according to - # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#about-caching-workflow-dependencies - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Populate lab cert.pfx - # https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#storing-base64-binary-blobs-as-secrets - run: echo $LAB_APP_CLIENT_CERT_BASE64 | base64 -d > $LAB_APP_CLIENT_CERT_PFX_PATH - - name: Test with pytest - run: pytest --benchmark-skip - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - #flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - #flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - cb: - # Benchmark only after the correctness has been tested by CI, - # and then run benchmark only once (sampling with only one Python version). - needs: ci - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - cache: 'pip' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Setup an updatable cache for Performance Baselines - uses: actions/cache@v4 - with: - path: .perf.baseline - key: ${{ runner.os }}-performance-${{ hashFiles('tests/test_benchmark.py') }} - restore-keys: ${{ runner.os }}-performance- - - name: Run benchmark - run: pytest --benchmark-only --benchmark-json benchmark.json --log-cli-level INFO tests/test_benchmark.py - - name: Render benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'pytest' - output-file-path: benchmark.json - fail-on-alert: true - - name: Publish Gibhub Pages - run: git push origin gh-pages - cd: - needs: ci # Note: github.event.pull_request.draft == false WON'T WORK in "if" statement, # because the triggered event is a push, not a pull_request. # This means each commit will trigger a release on TestPyPI. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b800165e..bc764fae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,3 +24,46 @@ stages: - template: .Pipelines/template-pipeline-stages.yml parameters: runPublish: false + +- stage: Benchmark + displayName: 'Run benchmarks' + dependsOn: CI + # Only run on post-merge pushes to dev — not on PRs or scheduled runs. + # Benchmarks are noisy and the baseline cache is only meaningful on a stable branch. + condition: | + and( + succeeded('CI'), + eq(variables['Build.Reason'], 'IndividualCI') + ) + jobs: + - job: Benchmark + displayName: 'Performance benchmarks (Python 3.9)' + pool: + vmImage: ubuntu-latest + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + displayName: 'Set up Python 3.9' + + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + + - task: Cache@2 + displayName: 'Restore performance baseline cache' + inputs: + key: '"perf-baseline" | "$(Agent.OS)" | tests/test_benchmark.py' + path: .perf.baseline + + - bash: | + pytest --benchmark-only --benchmark-json benchmark.json --log-cli-level INFO tests/test_benchmark.py + displayName: 'Run benchmarks' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish benchmark results' + condition: succeededOrFailed() + inputs: + PathtoPublish: 'benchmark.json' + ArtifactName: 'benchmark-results' From 10814c4e0bfe405d7fd4739a11e8ac66be3ca811 Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Tue, 31 Mar 2026 14:23:35 -0700 Subject: [PATCH 04/11] =?UTF-8?q?Remove=20GH=20Actions=20CI/CD=20workflow?= =?UTF-8?q?=20=E2=80=94=20tests=20and=20publish=20fully=20covered=20by=20A?= =?UTF-8?q?DO=20pipelines=203064=20and=203067?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/python-package.yml | 47 ---------------------------- 1 file changed, 47 deletions(-) delete mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index 2fe09625..00000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,47 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: CI/CD - -on: - push: - -jobs: - cd: - # Note: github.event.pull_request.draft == false WON'T WORK in "if" statement, - # because the triggered event is a push, not a pull_request. - # This means each commit will trigger a release on TestPyPI. - # Those releases will only succeed when each push has a new version number: a1, a2, a3, etc. - if: | - github.event_name == 'push' && - ( - startsWith(github.ref, 'refs/tags') || - startsWith(github.ref, 'refs/heads/release-') - ) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - cache: 'pip' - - name: Build a package for release - run: | - python -m pip install build --user - python -m build --sdist --wheel --outdir dist/ . - - name: | - Publish to TestPyPI when pushing to release-* branch. - You better test with a1, a2, b1, b2 releases first. - uses: pypa/gh-action-pypi-publish@v1.13.0 - if: startsWith(github.ref, 'refs/heads/release-') - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ - - name: Publish to PyPI when tagged - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@v1.13.0 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} From a8154b4c7c81967ff6aa196aa6e8ff5ecac91a23 Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Tue, 31 Mar 2026 14:46:22 -0700 Subject: [PATCH 05/11] Address Copilot review: fix benchmark branch condition, switch to PublishPipelineArtifact@1; add TSA config, PostBuildCleanup, job retries --- .Pipelines/template-pipeline-stages.yml | 9 +++++++++ .Pipelines/tsaConfig.json | 17 +++++++++++++++++ azure-pipelines.yml | 13 +++++++++---- 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 .Pipelines/tsaConfig.json diff --git a/.Pipelines/template-pipeline-stages.yml b/.Pipelines/template-pipeline-stages.yml index 30984da7..0b2db463 100644 --- a/.Pipelines/template-pipeline-stages.yml +++ b/.Pipelines/template-pipeline-stages.yml @@ -68,6 +68,14 @@ stages: GdnBreakGdnToolCredScan: true GdnBreakGdnToolPoliCheck: true + - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3 + displayName: 'Publish Security Analysis Logs (TSA)' + condition: succeededOrFailed() + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean agent directories' + condition: always() + # ══════════════════════════════════════════════════════════════════════════════ # Stage 1 · Validate — verify packageVersion matches msal/sku.py __version__ # Skipped when runPublish is false (PR / merge builds). @@ -123,6 +131,7 @@ stages: jobs: - job: Test displayName: 'Run unit tests' + retries: 3 pool: vmImage: ubuntu-latest strategy: diff --git a/.Pipelines/tsaConfig.json b/.Pipelines/tsaConfig.json new file mode 100644 index 00000000..f20b3111 --- /dev/null +++ b/.Pipelines/tsaConfig.json @@ -0,0 +1,17 @@ +{ + "codebaseName": "MSAL Python", + "notificationAliases": [ + "IdentityDevExDotnet@microsoft.com" + ], + "codebaseAdmins": [ + "EUROPE\\aadidagt" + ], + "instanceUrl": "https://identitydivision.visualstudio.com", + "projectName": "IDDP", + "areaPath": "IDDP\\DevEx-Client-SDK\\Python", + "iterationPath": "IDDP\\Unscheduled", + "tools": [ + "credscan", + "policheck" + ] +} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bc764fae..edc9f9d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,7 +33,12 @@ stages: condition: | and( succeeded('CI'), - eq(variables['Build.Reason'], 'IndividualCI') + eq(variables['Build.SourceBranch'], 'refs/heads/dev'), + or( + eq(variables['Build.Reason'], 'IndividualCI'), + eq(variables['Build.Reason'], 'BatchedCI'), + eq(variables['Build.Reason'], 'Manual') + ) ) jobs: - job: Benchmark @@ -61,9 +66,9 @@ stages: pytest --benchmark-only --benchmark-json benchmark.json --log-cli-level INFO tests/test_benchmark.py displayName: 'Run benchmarks' - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@1 displayName: 'Publish benchmark results' condition: succeededOrFailed() inputs: - PathtoPublish: 'benchmark.json' - ArtifactName: 'benchmark-results' + targetPath: 'benchmark.json' + artifact: 'benchmark-results' From 6107946745a4b823f2db37e0dbcf020bb2521e34 Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Tue, 31 Mar 2026 15:16:59 -0700 Subject: [PATCH 06/11] Fix: move retries from job to retryCountOnTaskFailure on pytest step (matrix jobs don't support retries) --- .Pipelines/template-pipeline-stages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.Pipelines/template-pipeline-stages.yml b/.Pipelines/template-pipeline-stages.yml index 0b2db463..2519f27e 100644 --- a/.Pipelines/template-pipeline-stages.yml +++ b/.Pipelines/template-pipeline-stages.yml @@ -131,7 +131,6 @@ stages: jobs: - job: Test displayName: 'Run unit tests' - retries: 3 pool: vmImage: ubuntu-latest strategy: @@ -198,6 +197,7 @@ stages: set -o pipefail pytest -vv --junitxml=test-results/junit.xml 2>&1 | tee test-results/pytest.log displayName: 'Run tests' + retryCountOnTaskFailure: 3 env: LAB_APP_CLIENT_CERT_PFX_PATH: $(LAB_APP_CLIENT_CERT_PFX_PATH) From 1bd1f0a94a79020c06194f12e186c35a7318235a Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Tue, 31 Mar 2026 16:43:36 -0700 Subject: [PATCH 07/11] Address Copilot review: wire tsaConfig.json to PublishSecurityAnalysisLogs@3; fix Cache@2 key escaping --- .Pipelines/template-pipeline-stages.yml | 2 ++ azure-pipelines.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.Pipelines/template-pipeline-stages.yml b/.Pipelines/template-pipeline-stages.yml index 2519f27e..b3a88741 100644 --- a/.Pipelines/template-pipeline-stages.yml +++ b/.Pipelines/template-pipeline-stages.yml @@ -71,6 +71,8 @@ stages: - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3 displayName: 'Publish Security Analysis Logs (TSA)' condition: succeededOrFailed() + inputs: + tsaConfigFile: '$(Build.SourcesDirectory)/.Pipelines/tsaConfig.json' - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean agent directories' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index edc9f9d4..a56a2428 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,7 +59,7 @@ stages: - task: Cache@2 displayName: 'Restore performance baseline cache' inputs: - key: '"perf-baseline" | "$(Agent.OS)" | tests/test_benchmark.py' + key: 'perf-baseline | "$(Agent.OS)" | tests/test_benchmark.py' path: .perf.baseline - bash: | From de03197d8c3dfba13a42d88a05c43d0baf1448da Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Fri, 8 May 2026 11:29:13 -0700 Subject: [PATCH 08/11] Move cryptography version-gating tests to warning-only step The cryptography ceiling tests fail whenever a new major version of the cryptography package is released. These are maintenance-reminder tests and should not block unrelated PRs. Run them in a separate step with continueOnError: true so they appear as warnings rather than failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .Pipelines/template-pipeline-stages.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.Pipelines/template-pipeline-stages.yml b/.Pipelines/template-pipeline-stages.yml index b3a88741..b3833934 100644 --- a/.Pipelines/template-pipeline-stages.yml +++ b/.Pipelines/template-pipeline-stages.yml @@ -197,12 +197,20 @@ stages: pip install pytest pytest-azurepipelines mkdir -p test-results set -o pipefail - pytest -vv --junitxml=test-results/junit.xml 2>&1 | tee test-results/pytest.log + pytest -vv --junitxml=test-results/junit.xml --deselect tests/test_cryptography.py::CryptographyTestCase::test_ceiling_should_be_latest_cryptography_version_plus_three --deselect tests/test_cryptography.py::CryptographyTestCase::test_should_be_run_with_latest_version_of_cryptography 2>&1 | tee test-results/pytest.log displayName: 'Run tests' retryCountOnTaskFailure: 3 env: LAB_APP_CLIENT_CERT_PFX_PATH: $(LAB_APP_CLIENT_CERT_PFX_PATH) + # Run cryptography version-gating tests separately as a warning-only check. + # These tests fail when a new cryptography major version is released and setup.cfg + # needs its upper bound bumped. They should not block unrelated PRs. + - bash: | + pytest -vv tests/test_cryptography.py::CryptographyTestCase::test_ceiling_should_be_latest_cryptography_version_plus_three tests/test_cryptography.py::CryptographyTestCase::test_should_be_run_with_latest_version_of_cryptography --junitxml=test-results/junit-crypto-ceiling.xml + displayName: 'Check cryptography ceiling (warning only)' + continueOnError: true + - task: PublishTestResults@2 displayName: 'Publish test results' condition: succeededOrFailed() @@ -212,6 +220,15 @@ stages: failTaskOnFailedTests: true testRunTitle: 'Python $(python.version)' + - task: PublishTestResults@2 + displayName: 'Publish cryptography ceiling results' + condition: succeededOrFailed() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'test-results/junit-crypto-ceiling.xml' + failTaskOnFailedTests: false + testRunTitle: 'Cryptography ceiling check $(python.version)' + - bash: rm -f "$(Agent.TempDirectory)/lab-auth.pfx" displayName: 'Clean up lab certificate' condition: always() From 358e010a51889d04c6bd9fefc81fe9177b9556bb Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Fri, 8 May 2026 13:00:41 -0700 Subject: [PATCH 09/11] Add Benchmark stage to pipeline documentation Update pipeline purpose, stage diagram, and stage table to include the Benchmark stage that runs on post-merge pushes to dev. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .Pipelines/CI-AND-RELEASE-PIPELINES.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.Pipelines/CI-AND-RELEASE-PIPELINES.md b/.Pipelines/CI-AND-RELEASE-PIPELINES.md index e7c5ad1a..253b7be1 100644 --- a/.Pipelines/CI-AND-RELEASE-PIPELINES.md +++ b/.Pipelines/CI-AND-RELEASE-PIPELINES.md @@ -9,7 +9,7 @@ including what each pipeline does, when it runs, and how to trigger a release. | File | ADO Pipeline | Purpose | |------|-------------|---------| -| [`azure-pipelines.yml`](../azure-pipelines.yml) | [MSAL.Python-PR-OneBranch-Official (3064)](https://dev.azure.com/IdentityDivision/IDDP/_build?definitionId=3064) | PR gate and post-merge CI — calls the shared template with `runPublish: false` | +| [`azure-pipelines.yml`](../azure-pipelines.yml) | [MSAL.Python-PR-OneBranch-Official (3064)](https://dev.azure.com/IdentityDivision/IDDP/_build?definitionId=3064) | PR gate, post-merge CI, and performance benchmarks — calls the shared template with `runPublish: false`; runs benchmarks on post-merge pushes to `dev` | | [`pipeline-publish.yml`](pipeline-publish.yml) | [MSAL.Python-Publish (3067)](https://dev.azure.com/IdentityDivision/IDDP/_build?definitionId=3067) | Release pipeline — manually queued, builds and publishes to PyPI | | [`template-pipeline-stages.yml`](template-pipeline-stages.yml) | — | Shared stages template — PreBuildCheck, Validate, and CI stages reused by both pipelines | | [`credscan-exclusion.json`](credscan-exclusion.json) | — | CredScan suppression file for known test fixtures | @@ -29,13 +29,14 @@ including what each pipeline does, when it runs, and how to trigger a release. ### Stages ``` -PreBuildCheck ─► CI +PreBuildCheck ─► CI ─► Benchmark (post-merge to dev only) ``` -| Stage | What it does | -|-------|-------------| -| **PreBuildCheck** | Runs SDL security scans: PoliCheck (policy/offensive content), CredScan (leaked credentials), and PostAnalysis (breaks the build on findings) | -| **CI** | Runs the full test suite on Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14 | +| Stage | What it does | When it runs | +|-------|-------------|-------------| +| **PreBuildCheck** | Runs SDL security scans: PoliCheck (policy/offensive content), CredScan (leaked credentials), and PostAnalysis (breaks the build on findings) | Always | +| **CI** | Runs the full test suite on Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, and 3.14 | Always | +| **Benchmark** | Runs performance benchmarks on Python 3.9 and publishes `benchmark-results` artifact | Post-merge pushes to `dev` and manual runs only | The Validate stage is **skipped** on PR/CI runs (it only applies to release builds). From cfbd078227683e4b2518d87f75658fc226e3a788 Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Fri, 8 May 2026 13:47:16 -0700 Subject: [PATCH 10/11] Address Copilot review: benchmark skip, pytest install, artifact guard - Add --benchmark-skip to CI pytest runs so benchmarks only run in the dedicated Benchmark stage, not on every PR across all Python versions - Explicitly install pytest in the Benchmark job instead of relying on transitive dependency from pytest-benchmark - Guard PublishPipelineArtifact with a file existence check so a missing benchmark.json does not mask the underlying failure Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .Pipelines/template-pipeline-stages.yml | 2 +- azure-pipelines.yml | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.Pipelines/template-pipeline-stages.yml b/.Pipelines/template-pipeline-stages.yml index b3833934..b411fa0a 100644 --- a/.Pipelines/template-pipeline-stages.yml +++ b/.Pipelines/template-pipeline-stages.yml @@ -197,7 +197,7 @@ stages: pip install pytest pytest-azurepipelines mkdir -p test-results set -o pipefail - pytest -vv --junitxml=test-results/junit.xml --deselect tests/test_cryptography.py::CryptographyTestCase::test_ceiling_should_be_latest_cryptography_version_plus_three --deselect tests/test_cryptography.py::CryptographyTestCase::test_should_be_run_with_latest_version_of_cryptography 2>&1 | tee test-results/pytest.log + pytest -vv --benchmark-skip --junitxml=test-results/junit.xml --deselect tests/test_cryptography.py::CryptographyTestCase::test_ceiling_should_be_latest_cryptography_version_plus_three --deselect tests/test_cryptography.py::CryptographyTestCase::test_should_be_run_with_latest_version_of_cryptography 2>&1 | tee test-results/pytest.log displayName: 'Run tests' retryCountOnTaskFailure: 3 env: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a56a2428..5665065e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,7 +53,7 @@ stages: - script: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install pytest -r requirements.txt displayName: 'Install dependencies' - task: Cache@2 @@ -66,9 +66,14 @@ stages: pytest --benchmark-only --benchmark-json benchmark.json --log-cli-level INFO tests/test_benchmark.py displayName: 'Run benchmarks' + - bash: | + [ -f benchmark.json ] && echo "##vso[task.setvariable variable=benchmarkFileExists]true" + displayName: 'Check benchmark output exists' + condition: succeededOrFailed() + - task: PublishPipelineArtifact@1 displayName: 'Publish benchmark results' - condition: succeededOrFailed() + condition: and(succeededOrFailed(), eq(variables['benchmarkFileExists'], 'true')) inputs: targetPath: 'benchmark.json' artifact: 'benchmark-results' From 30bd1069bca26af80e78a2c93d05c93c2f81592f Mon Sep 17 00:00:00 2001 From: Ryan Auld Date: Thu, 14 May 2026 13:41:49 -0700 Subject: [PATCH 11/11] Fix: allow Build/Publish stages to proceed on SucceededWithIssues The cryptography ceiling check runs with continueOnError: true, which causes the CI stage to finish as SucceededWithIssues rather than Succeeded. The strict eq(..., 'Succeeded') conditions on Build, PublishMSALPython, and PublishPyPI stages caused them to skip. Change all three conditions to use in(..., 'Succeeded', 'SucceededWithIssues') so warning-only tasks don't block the release. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .Pipelines/pipeline-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.Pipelines/pipeline-publish.yml b/.Pipelines/pipeline-publish.yml index 818dfddb..f2926a4c 100644 --- a/.Pipelines/pipeline-publish.yml +++ b/.Pipelines/pipeline-publish.yml @@ -44,7 +44,7 @@ stages: - stage: Build displayName: 'Build package' dependsOn: CI - condition: eq(dependencies.CI.result, 'Succeeded') + condition: in(dependencies.CI.result, 'Succeeded', 'SucceededWithIssues') jobs: - job: BuildDist displayName: 'Build sdist + wheel (Python 3.12)' @@ -83,7 +83,7 @@ stages: dependsOn: Build condition: > and( - eq(dependencies.Build.result, 'Succeeded'), + in(dependencies.Build.result, 'Succeeded', 'SucceededWithIssues'), eq('${{ parameters.publishTarget }}', 'test.pypi.org (Preview / RC)') ) jobs: @@ -140,7 +140,7 @@ stages: dependsOn: Build condition: > and( - eq(dependencies.Build.result, 'Succeeded'), + in(dependencies.Build.result, 'Succeeded', 'SucceededWithIssues'), eq('${{ parameters.publishTarget }}', 'pypi.org (ESRP Production)') ) jobs: