From e1d2cc79fe1f3e48bc7b5054ccd12199c0df9d04 Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Fri, 13 Mar 2026 10:03:02 -0700 Subject: [PATCH 1/2] Update common Docker engineering infrastructure with latest --- eng/docker-tools/CHANGELOG.md | 31 ++++++++ eng/docker-tools/templates/1es-official.yml | 8 --- eng/docker-tools/templates/1es-unofficial.yml | 9 --- .../templates/jobs/build-images.yml | 11 +++ .../templates/jobs/copy-base-images.yml | 5 ++ .../templates/jobs/generate-matrix.yml | 7 ++ eng/docker-tools/templates/jobs/publish.yml | 11 +++ .../templates/jobs/sign-images.yml | 6 ++ .../stages/dotnet/build-test-publish-repo.yml | 5 ++ .../templates/stages/dotnet/publish.yml | 4 ++ eng/docker-tools/templates/stages/publish.yml | 5 ++ .../steps/reference-service-connections.yml | 72 +++++++++++++++++++ .../steps/test-images-linux-client.yml | 8 ++- .../templates/variables/docker-images.yml | 2 +- 14 files changed, 165 insertions(+), 19 deletions(-) create mode 100644 eng/docker-tools/templates/steps/reference-service-connections.yml diff --git a/eng/docker-tools/CHANGELOG.md b/eng/docker-tools/CHANGELOG.md index aba13609..480b25ac 100644 --- a/eng/docker-tools/CHANGELOG.md +++ b/eng/docker-tools/CHANGELOG.md @@ -4,14 +4,45 @@ All breaking changes and new features in `eng/docker-tools` will be documented i --- +## 2026-03-12: Service connection OIDC changes + +- Pull request: [#2013](https://github.com/dotnet/docker-tools/pull/2013) +- Issue: [#2012](https://github.com/dotnet/docker-tools/issues/2012) + +`setup-service-connections.yml` has been removed. Azure DevOps no longer +issues OIDC tokens for service connections referenced in a separate stage. +Service connections are now referenced per-job via +`reference-service-connections.yml`. + +**How to update:** + +- Remove any `serviceConnections` parameters passed to `1es-official.yml` or + `1es-unofficial.yml` - they are no longer accepted. +- Remove any calls to `setup-service-connections.yml` from stage templates. +- Non-registry service connections (e.g., kusto, marStatus) should be passed + via `additionalServiceConnections` to the job templates that need them. + +--- + ## 2026-03-04: Pre-build validation gated by `preBuildTestScriptPath` variable +- Pull request: [#1997](https://github.com/dotnet/docker-tools/pull/1997) + The `PreBuildValidation` job condition now checks the new `preBuildTestScriptPath` variable instead of `testScriptPath`. This allows repos to independently control whether pre-build validation runs, without affecting functional tests. The new variable defaults to `$(testScriptPath)`, so existing repos that have pre-build tests are not affected. Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely. +### Update (2026-03-11): Use `preBuildTestScriptPath` for test execution + +- Pull request: [#2011](https://github.com/dotnet/docker-tools/pull/2011) + +The `PreBuildValidation` job now uses `preBuildTestScriptPath` for test execution instead of `testScriptPath`. +Previously, the job condition was gated on `preBuildTestScriptPath` but the test execution step still used `testScriptPath`, +which meant PreBuildValidation could not be enabled independently when `testScriptPath` was empty. +Repos that do not have pre-build tests can set `preBuildTestScriptPath` to `""` to skip the job entirely. + --- ## 2026-02-19: Separate Registry Endpoints from Authentication diff --git a/eng/docker-tools/templates/1es-official.yml b/eng/docker-tools/templates/1es-official.yml index ebf8fcd7..8eb33d65 100644 --- a/eng/docker-tools/templates/1es-official.yml +++ b/eng/docker-tools/templates/1es-official.yml @@ -17,9 +17,6 @@ parameters: - name: stages type: stageList default: [] -- name: serviceConnections - type: object - default: [] - name: pool type: object default: @@ -62,9 +59,4 @@ extends: tsa: enabled: true stages: - - ${{ if gt(length(parameters.serviceConnections), 0) }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - pool: ${{ parameters.pool }} - serviceConnections: ${{ parameters.serviceConnections }} - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/1es-unofficial.yml b/eng/docker-tools/templates/1es-unofficial.yml index bc584cd3..c4a849c5 100644 --- a/eng/docker-tools/templates/1es-unofficial.yml +++ b/eng/docker-tools/templates/1es-unofficial.yml @@ -19,10 +19,6 @@ parameters: - name: stages type: stageList default: [] - # 1ES Pipeline Template parameters -- name: serviceConnections - type: object - default: [] - name: pool type: object default: @@ -71,9 +67,4 @@ extends: tsa: enabled: true stages: - - ${{ if gt(length(parameters.serviceConnections), 0) }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - pool: ${{ parameters.pool }} - serviceConnections: ${{ parameters.serviceConnections }} - ${{ parameters.stages }} diff --git a/eng/docker-tools/templates/jobs/build-images.yml b/eng/docker-tools/templates/jobs/build-images.yml index 025d3c8d..7327b6d6 100644 --- a/eng/docker-tools/templates/jobs/build-images.yml +++ b/eng/docker-tools/templates/jobs/build-images.yml @@ -42,6 +42,17 @@ jobs: cleanupDocker: true customInitSteps: ${{ parameters.customInitSteps }} - ${{ parameters.customBuildInitSteps }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + dockerClientOS: ${{ parameters.dockerClientOS }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + # Check .name instead of the whole object - null parameters can become + # empty objects through template layers, making ${{ if }} truthy. + ${{ if parameters.storageAccountServiceConnection.name }}: + serviceConnections: + - name: ${{ parameters.storageAccountServiceConnection.name }} - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self parameters: publicSourceBranch: $(publicSourceBranch) diff --git a/eng/docker-tools/templates/jobs/copy-base-images.yml b/eng/docker-tools/templates/jobs/copy-base-images.yml index c435b211..4a5ed5e8 100644 --- a/eng/docker-tools/templates/jobs/copy-base-images.yml +++ b/eng/docker-tools/templates/jobs/copy-base-images.yml @@ -43,6 +43,11 @@ jobs: publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customInitSteps }} versionsRepoRef: ${{ parameters.versionsRepoRef }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.acr.server }} - ${{ parameters.customCopyBaseImagesInitSteps }} - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self parameters: diff --git a/eng/docker-tools/templates/jobs/generate-matrix.yml b/eng/docker-tools/templates/jobs/generate-matrix.yml index 8d22b391..fe668c65 100644 --- a/eng/docker-tools/templates/jobs/generate-matrix.yml +++ b/eng/docker-tools/templates/jobs/generate-matrix.yml @@ -25,6 +25,13 @@ jobs: publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} + # When --trim-cached-images is active, ImageBuilder checks base image digests + # in the ACR mirror registry, which requires OIDC auth via this service connection. + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} - ${{ parameters.customGenerateMatrixInitSteps }} - template: /eng/docker-tools/templates/steps/retain-build.yml@self - template: /eng/docker-tools/templates/steps/validate-branch.yml@self diff --git a/eng/docker-tools/templates/jobs/publish.yml b/eng/docker-tools/templates/jobs/publish.yml index b86ec1ee..92f98b22 100644 --- a/eng/docker-tools/templates/jobs/publish.yml +++ b/eng/docker-tools/templates/jobs/publish.yml @@ -12,6 +12,9 @@ parameters: # When true, overrides the commit SHA in merged image info files to use the current repository commit. # This ensures that updated images reference the correct commit in their commitUrl properties. overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] jobs: - job: Publish @@ -53,6 +56,14 @@ jobs: versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customInitSteps }} + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + - ${{ parameters.publishConfig.PublishRegistry.server }} + serviceConnections: ${{ parameters.additionalServiceConnections }} + - template: /eng/docker-tools/templates/steps/retain-build.yml@self - pwsh: | diff --git a/eng/docker-tools/templates/jobs/sign-images.yml b/eng/docker-tools/templates/jobs/sign-images.yml index 8162cf54..fcc43f09 100644 --- a/eng/docker-tools/templates/jobs/sign-images.yml +++ b/eng/docker-tools/templates/jobs/sign-images.yml @@ -30,6 +30,12 @@ jobs: publishConfig: ${{ parameters.publishConfig }} envFilePath: $(signingEnvFilePath) + - template: /eng/docker-tools/templates/steps/reference-service-connections.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} + usesRegistries: + - ${{ parameters.publishConfig.BuildRegistry.server }} + # Download merged image-info artifact from Post_Build stage (or from a previous pipeline run) - template: /eng/docker-tools/templates/steps/download-build-artifact.yml@self parameters: diff --git a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml index 13ca93a3..c590a528 100644 --- a/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml @@ -32,6 +32,10 @@ parameters: # Publish parameters customPublishInitSteps: [] + # Additional service connections not in publishConfig.RegistryAuthentication + # that need OIDC token access (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + # Other common parameters internalProjectName: null publicProjectName: null @@ -75,5 +79,6 @@ stages: internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} publishConfig: ${{ parameters.publishConfig }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/docker-tools/templates/stages/dotnet/publish.yml b/eng/docker-tools/templates/stages/dotnet/publish.yml index ae0325f0..5cac237b 100644 --- a/eng/docker-tools/templates/stages/dotnet/publish.yml +++ b/eng/docker-tools/templates/stages/dotnet/publish.yml @@ -13,6 +13,9 @@ parameters: sourceBuildPipelineRunId: '' versionsRepoRef: null overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] stages: - template: /eng/docker-tools/templates/stages/publish.yml@self @@ -22,6 +25,7 @@ stages: publishConfig: ${{ parameters.publishConfig }} isStandalonePublish: ${{ parameters.isStandalonePublish }} customInitSteps: ${{ parameters.customInitSteps }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/docker-tools/templates/stages/publish.yml b/eng/docker-tools/templates/stages/publish.yml index f29c376c..7195aaa8 100644 --- a/eng/docker-tools/templates/stages/publish.yml +++ b/eng/docker-tools/templates/stages/publish.yml @@ -25,6 +25,10 @@ parameters: # internally built images still reference public Dockerfiles. overrideImageInfoCommit: false + # Service connections not in publishConfig.RegistryAuthentication that need OIDC + # token access during publish (e.g., kusto, marStatus). Shape: [{ name: string }] + additionalServiceConnections: [] + ################################################################################ # Publish Images ################################################################################ @@ -77,3 +81,4 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} versionsRepoPath: ${{ parameters.versionsRepoPath }} overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} diff --git a/eng/docker-tools/templates/steps/reference-service-connections.yml b/eng/docker-tools/templates/steps/reference-service-connections.yml new file mode 100644 index 00000000..ab6ec980 --- /dev/null +++ b/eng/docker-tools/templates/steps/reference-service-connections.yml @@ -0,0 +1,72 @@ +# Emit AzureCLI@2 steps that reference service connections in the current +# stage so that Azure DevOps will issue OIDC tokens for them. Azure DevOps +# requires each stage to explicitly reference (via an azureSubscription task +# input) any service connection it needs OIDC tokens for. +# +# This template must be included in every job that authenticates to Azure +# via AzurePipelinesCredential (i.e., any job that uses run-imagebuilder.yml +# with internalProjectName, or run-pwsh-with-auth.yml). +# +# Service connections can be specified in two ways: +# - Via publishConfig + usesRegistries: looks up service connections from +# publishConfig.RegistryAuthentication entries matching the given servers. +# Use this for registry-scoped connections (e.g., BuildRegistry, PublishRegistry). +# - Via serviceConnections: a direct list of { name: string } objects. +# Use this for non-registry connections (e.g., kusto, marStatus, cleanServiceConnection). +# +# Both can be used together in a single call. +parameters: +# Publishing configuration object. Only needed when using the usesRegistries parameter. +- name: publishConfig + type: object + default: {} +# List of registry server URLs to look up in publishConfig.RegistryAuthentication. +# Each matching entry's service connection will be referenced. +- name: usesRegistries + type: object + default: [] +# Direct list of service connections to reference. Shape: [{ name: string }] +- name: serviceConnections + type: object + default: [] +# The OS of the agent running this job. Determines whether to use PowerShell +# Core (pscore, Linux) or Windows PowerShell (ps, Windows) for the AzureCLI task. +- name: dockerClientOS + type: string + default: linux +# The internal Azure DevOps project name. Reference steps are only emitted +# for internal non-PR builds, since public projects don't have these service +# connections. +- name: internalProjectName + type: string + default: internal + +steps: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + # Guard on .name: null parameters passed through template layers can become + # empty objects that are truthy, so check the concrete property instead. + - ${{ each serviceConnection in parameters.serviceConnections }}: + - ${{ if serviceConnection.name }}: + - task: AzureCLI@2 + displayName: Reference ${{ serviceConnection.name }} + inputs: + azureSubscription: ${{ serviceConnection.name }} + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: ps + ${{ else }}: + scriptType: pscore + scriptLocation: inlineScript + inlineScript: Write-Host "Service connection referenced for OIDC" + - ${{ each auth in parameters.publishConfig.RegistryAuthentication }}: + # Also guard on .name here for the same reason as the serviceConnections loop above. + - ${{ if and(containsValue(parameters.usesRegistries, auth.server), auth.serviceConnection.name) }}: + - task: AzureCLI@2 + displayName: Reference ${{ auth.serviceConnection.name }} + inputs: + azureSubscription: ${{ auth.serviceConnection.name }} + ${{ if eq(parameters.dockerClientOS, 'windows') }}: + scriptType: ps + ${{ else }}: + scriptType: pscore + scriptLocation: inlineScript + inlineScript: Write-Host "Service connection referenced for OIDC" diff --git a/eng/docker-tools/templates/steps/test-images-linux-client.yml b/eng/docker-tools/templates/steps/test-images-linux-client.yml index 8f0e8426..1806dbb7 100644 --- a/eng/docker-tools/templates/steps/test-images-linux-client.yml +++ b/eng/docker-tools/templates/steps/test-images-linux-client.yml @@ -25,6 +25,12 @@ steps: - script: | echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)" + if [ "${{ parameters.preBuildValidation }}" == "true" ]; then + echo "##vso[task.setvariable variable=effectiveTestScriptPath]$(preBuildTestScriptPath)" + else + echo "##vso[task.setvariable variable=effectiveTestScriptPath]$(testScriptPath)" + fi + additionalTestArgs="$ADDITIONALTESTARGS" if [ "${{ parameters.preBuildValidation }}" == "true" ]; then additionalTestArgs="$additionalTestArgs -TestCategories pre-build" @@ -74,7 +80,7 @@ steps: $(testRunner.options) $(testRunner.container) pwsh - -Command "$(testScriptPath) + -Command "$(effectiveTestScriptPath) -Paths $(imageBuilderPathsArrayInitStr) -OSVersions $(osVersionsArrayInitStr) -Architecture '$(architecture)' diff --git a/eng/docker-tools/templates/variables/docker-images.yml b/eng/docker-tools/templates/variables/docker-images.yml index 293657be..73aa4051 100644 --- a/eng/docker-tools/templates/variables/docker-images.yml +++ b/eng/docker-tools/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2919324 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2923333 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner From 11aaaecc3f4f6687a8c83d2bb77d3a17a466b940 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 13 Mar 2026 10:47:41 -0700 Subject: [PATCH 2/2] Update pipelines in response to eng/docker-tools breaking changes --- eng/pipelines/stages/dotnet-framework-base.yml | 13 +------------ eng/pipelines/update-readmes.yml | 4 +++- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/eng/pipelines/stages/dotnet-framework-base.yml b/eng/pipelines/stages/dotnet-framework-base.yml index b7501e5e..f5529ce9 100644 --- a/eng/pipelines/stages/dotnet-framework-base.yml +++ b/eng/pipelines/stages/dotnet-framework-base.yml @@ -23,18 +23,6 @@ parameters: stages: -- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/docker-tools/templates/stages/setup-service-connections.yml@self - parameters: - publishConfig: ${{ parameters.publishConfig }} - usesRegistries: - - ${{ parameters.publishConfig.InternalMirrorRegistry.server }} - - ${{ parameters.publishConfig.BuildRegistry.server }} - - ${{ parameters.publishConfig.PublishRegistry.server }} - serviceConnections: - - ${{ each serviceConnection in parameters.additionalServiceConnections }}: - - name: ${{ serviceConnection.name }} - - template: /eng/docker-tools/templates/stages/dotnet/build-test-publish-repo.yml@self parameters: publishConfig: ${{ parameters.publishConfig }} @@ -43,6 +31,7 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} noCache: ${{ parameters.noCache }} + additionalServiceConnections: ${{ parameters.additionalServiceConnections }} ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: windowsAmdBuildJobTimeout: 330 diff --git a/eng/pipelines/update-readmes.yml b/eng/pipelines/update-readmes.yml index c629c767..dd0a313a 100644 --- a/eng/pipelines/update-readmes.yml +++ b/eng/pipelines/update-readmes.yml @@ -16,6 +16,8 @@ extends: - job: UpdateReadmes displayName: Update Readmes steps: - - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self + - template: /eng/docker-tools/templates/steps/init-common.yml@self + parameters: + dockerClientOS: linux - template: /eng/docker-tools/templates/steps/publish-readmes.yml@self - template: /eng/docker-tools/templates/steps/cleanup-docker-linux.yml@self