diff --git a/eng/pipelines/aspnetcore-perf-build-jobs.yml b/eng/pipelines/aspnetcore-perf-build-jobs.yml new file mode 100644 index 00000000000..6c9dcaac55d --- /dev/null +++ b/eng/pipelines/aspnetcore-perf-build-jobs.yml @@ -0,0 +1,364 @@ +# aspnetcore-perf-build-jobs.yml +# +# Authored build jobs for the ASP.NET Core perf-build pipeline (consumed by +# eng/pipelines/aspnetcore-perf-build.yml). These jobs are written from scratch +# here in dotnet/performance instead of reusing dotnet/aspnetcore's +# `.azure/pipelines/jobs/default-build.yml`, because that template has hard +# `@self` references (e.g. /eng/common/templates-official/job/job.yml@self) and +# `@self` always resolves to the ROOT pipeline repo (performance), which does not +# carry aspnetcore's eng/common layout/contract. So we stand up the build +# ourselves: pick the dnceng internal pool, check out the aspnetcore mirror at the +# triggering commit alongside this repo (for the staging script), run aspnetcore's +# own eng/build with lean perf args (PUBLIC feeds only -- no internal runtime +# download), stage each per-RID runtime-pack nupkg verbatim as the BCS artifact via +# the moved eng/pipelines/tools/stage-bcs-nupkg-aspnetcore.ps1, and publish one +# pipeline artifact per config. +# +# Job names (Windows_x64_build / Windows_x86_build / Windows_arm64_build / +# Linux_x64_build / Linux_arm64_build) are load-bearing: +# they must match the `dependencyJobName` values in +# eng/pipelines/upload-build-artifacts-jobs.yml's aspnetcore_* branches. +# +# PUBLIC-FEED rationale: aspnetcore's own ci-public.yml builds every public PR with +# `_InternalRuntimeDownloadArgs` empty, proving public feeds suffice for a from- +# source pack build. We therefore drop enable-internal-runtimes / get-delegation-sas +# / the dotnetbuilds-internal-read connection entirely and pass no +# -RuntimeSourceFeed args. + +parameters: + aspnetcoreRepoAlias: aspnetcore + performanceRepoAlias: self + # Per-config enablement booleans (forward-compat with the MissingBuildsTrigger + # PerConfiguration indexer). In v1's eager per-commit CI-trigger mode all five + # are true and all configs build; the booleans let a future Function queue a + # subset per config. + aspnetcore_x64_linux: true + aspnetcore_arm64_linux: true + aspnetcore_x64_windows: true + aspnetcore_arm64_windows: true + aspnetcore_x86_windows: true + +jobs: + +# =========================================================================== +# Windows build jobs (one self-sufficient job per arch, mirroring the Linux +# jobs below). Each job is independently gated by its config boolean, checks out +# the aspnetcore mirror + this repo, runs aspnetcore's eng/build.cmd standalone +# for its arch, packs the per-RID archive, and publishes one artifact. +# +# Unlike aspnetcore ci-public.yml -- which builds x64/x86/arm64 sequentially in +# ONE job so x86/arm64 reuse the x64 step's managed build and on-machine native +# toolchain -- these jobs run on separate agents, so each must be self-sufficient: +# * every arch carries -all (full managed build) and -nativeToolsOnMachine +# (set up the native toolchain the shared x64 step used to provide); +# * x86/arm64 keep -noBuildNative (faithful to ci-public's per-arch args; their +# native runtime bits restore from packages rather than cross-building on the +# x64 host). The first real queued run validates standalone native restore. +# +# Job names (Windows_x64_build / Windows_x86_build / Windows_arm64_build) are +# load-bearing: they must match the `dependencyJobName` values in +# eng/pipelines/upload-build-artifacts-jobs.yml's aspnetcore_*_windows branches. +# =========================================================================== +- ${{ if eq(parameters.aspnetcore_x64_windows, true) }}: + - job: Windows_x64_build + displayName: 'Build: Windows x64 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + variables: + _AspNetCoreRoot: $(Agent.BuildDirectory)\s\aspnetcore + _StageNupkgScript: $(Agent.BuildDirectory)\s\performance\eng\pipelines\tools\stage-bcs-nupkg-aspnetcore.ps1 + _ShippingDir: $(Agent.BuildDirectory)\s\aspnetcore\artifacts\packages\Release\Shipping + _StagingRoot: $(Build.ArtifactStagingDirectory)\bcs + MSBUILDUSESERVER: "1" + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + - script: >- + eng\build.cmd + -ci + -prepareMachine + -nativeToolsOnMachine + -Configuration Release + -arch x64 + -pack + -all + -noBuildJava + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.x64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build x64 + env: + MSBUILDUSESERVER: "1" + + - pwsh: >- + & "$(_StageNupkgScript)" -Rids win-x64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Stage BCS nupkg (win-x64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_windows_x64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)\BuildArtifacts_windows_x64_Release_aspnetcore + artifactName: BuildArtifacts_windows_x64_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Windows x64 build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)\artifacts\log + artifactName: Windows_x64_perf_build_Logs_Attempt_$(System.JobAttempt) + +# =========================================================================== +# Windows x86 build job +# =========================================================================== +- ${{ if eq(parameters.aspnetcore_x86_windows, true) }}: + - job: Windows_x86_build + displayName: 'Build: Windows x86 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + variables: + _AspNetCoreRoot: $(Agent.BuildDirectory)\s\aspnetcore + _StageNupkgScript: $(Agent.BuildDirectory)\s\performance\eng\pipelines\tools\stage-bcs-nupkg-aspnetcore.ps1 + _ShippingDir: $(Agent.BuildDirectory)\s\aspnetcore\artifacts\packages\Release\Shipping + _StagingRoot: $(Build.ArtifactStagingDirectory)\bcs + MSBUILDUSESERVER: "1" + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + - script: >- + eng\build.cmd + -ci + -prepareMachine + -nativeToolsOnMachine + -Configuration Release + -arch x86 + -pack + -all + -noBuildJava + -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.x86.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build x86 + env: + MSBUILDUSESERVER: "1" + + - pwsh: >- + & "$(_StageNupkgScript)" -Rids win-x86 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Stage BCS nupkg (win-x86)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_windows_x86_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)\BuildArtifacts_windows_x86_Release_aspnetcore + artifactName: BuildArtifacts_windows_x86_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Windows x86 build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)\artifacts\log + artifactName: Windows_x86_perf_build_Logs_Attempt_$(System.JobAttempt) + +# =========================================================================== +# Windows arm64 build job +# =========================================================================== +- ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}: + - job: Windows_arm64_build + displayName: 'Build: Windows ARM64 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + variables: + _AspNetCoreRoot: $(Agent.BuildDirectory)\s\aspnetcore + _StageNupkgScript: $(Agent.BuildDirectory)\s\performance\eng\pipelines\tools\stage-bcs-nupkg-aspnetcore.ps1 + _ShippingDir: $(Agent.BuildDirectory)\s\aspnetcore\artifacts\packages\Release\Shipping + _StagingRoot: $(Build.ArtifactStagingDirectory)\bcs + MSBUILDUSESERVER: "1" + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + - script: >- + eng\build.cmd + -ci + -prepareMachine + -nativeToolsOnMachine + -Configuration Release + -arch arm64 + -pack + -all + -noBuildJava + -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.arm64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build ARM64 + env: + MSBUILDUSESERVER: "1" + + - pwsh: >- + & "$(_StageNupkgScript)" -Rids win-arm64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Stage BCS nupkg (win-arm64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_windows_arm64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)\BuildArtifacts_windows_arm64_Release_aspnetcore + artifactName: BuildArtifacts_windows_arm64_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Windows arm64 build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)\artifacts\log + artifactName: Windows_arm64_perf_build_Logs_Attempt_$(System.JobAttempt) + +# =========================================================================== +# Linux x64 build job +# =========================================================================== +- ${{ if eq(parameters.aspnetcore_x64_linux, true) }}: + - job: Linux_x64_build + displayName: 'Build: Linux x64 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-ubuntu-2204 + variables: + _AspNetCoreRoot: $(Agent.BuildDirectory)/s/aspnetcore + _StageNupkgScript: $(Agent.BuildDirectory)/s/performance/eng/pipelines/tools/stage-bcs-nupkg-aspnetcore.ps1 + _ShippingDir: $(Agent.BuildDirectory)/s/aspnetcore/artifacts/packages/Release/Shipping + _StagingRoot: $(Build.ArtifactStagingDirectory)/bcs + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + - script: >- + eng/build.sh + --ci + --configuration Release + --arch x64 + --pack + --all + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.linux-x64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build linux-x64 + + - pwsh: >- + & "$(_StageNupkgScript)" -Rids linux-x64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Stage BCS nupkg (linux-x64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_linux_x64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)/BuildArtifacts_linux_x64_Release_aspnetcore + artifactName: BuildArtifacts_linux_x64_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Linux x64 build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)/artifacts/log + artifactName: Linux_x64_perf_build_Logs_Attempt_$(System.JobAttempt) + +# =========================================================================== +# Linux arm64 build job (cross-build from the x64 host) +# =========================================================================== +- ${{ if eq(parameters.aspnetcore_arm64_linux, true) }}: + - job: Linux_arm64_build + displayName: 'Build: Linux ARM64 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-ubuntu-2204 + variables: + _AspNetCoreRoot: $(Agent.BuildDirectory)/s/aspnetcore + _StageNupkgScript: $(Agent.BuildDirectory)/s/performance/eng/pipelines/tools/stage-bcs-nupkg-aspnetcore.ps1 + _ShippingDir: $(Agent.BuildDirectory)/s/aspnetcore/artifacts/packages/Release/Shipping + _StagingRoot: $(Build.ArtifactStagingDirectory)/bcs + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + - script: >- + eng/build.sh + --ci + --configuration Release + --arch arm64 + --pack + --all + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.linux-arm64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build linux-arm64 + + - pwsh: >- + & "$(_StageNupkgScript)" -Rids linux-arm64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Stage BCS nupkg (linux-arm64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_linux_arm64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)/BuildArtifacts_linux_arm64_Release_aspnetcore + artifactName: BuildArtifacts_linux_arm64_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Linux arm64 build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)/artifacts/log + artifactName: Linux_arm64_perf_build_Logs_Attempt_$(System.JobAttempt) diff --git a/eng/pipelines/aspnetcore-perf-build.yml b/eng/pipelines/aspnetcore-perf-build.yml new file mode 100644 index 00000000000..252cc7606b0 --- /dev/null +++ b/eng/pipelines/aspnetcore-perf-build.yml @@ -0,0 +1,193 @@ +# +# aspnetcore-perf-build.yml +# +# Azure DevOps pipeline (hosted in dotnet/performance) that builds every commit +# on dotnet/aspnetcore `main` from source, packs per-RID +# Microsoft.AspNetCore.App.Runtime archives, and uploads them to the Build Cache +# Service (BCS @ pvscmdupload) so dotnet/crank can resolve ASP.NET Core runtime +# binaries by commit SHA for performance-regression bisection. +# +# WHY THIS LIVES HERE (and not in dotnet/aspnetcore): moving it into +# dotnet/performance gives the perf team full merge control. The previous +# aspnetcore-hosted version reused aspnetcore's `default-build.yml`, which cannot +# be consumed cross-repo (its `@self` references resolve to the ROOT pipeline +# repo = performance). So the build jobs are re-authored from source in +# eng/pipelines/aspnetcore-perf-build-jobs.yml. +# +# HOW PER-COMMIT FIRING WORKS: the aspnetcore AzDO mirror +# (internal/dotnet-aspnetcore) is declared as a repository resource with a CI +# `trigger` on main. A push to aspnetcore main fires this pipeline +# (Build.Reason == ResourceTrigger). The build jobs check out that resource at +# the triggering commit. +# +# SHA INVERSION (load-bearing): because the pipeline is hosted in performance, +# `Build.SourceVersion` is the PERFORMANCE commit, not the aspnetcore one. The +# correct BCS {sha} is the triggering aspnetcore commit, exposed as +# `$(resources.repositories.aspnetcore.version)`. We pass it as `sha:` to the +# shared register/upload templates (which default `sha` to $(Build.SourceVersion) +# for the runtime perf-build that is hosted in dotnet/runtime). +# +# PUBLIC FEEDS ONLY: like aspnetcore's ci-public.yml, the from-source pack build +# uses public feeds (no internal runtime download, no dotnetbuilds-internal-read +# connection). Only the BCS upload uses the proven `.NET Performance` connection. +# +# NOT 1ES: per-commit perf artifacts go to a private BCS cache (unsigned, not +# redistributed), so we do not extend 1ES.Official -- matching dotnet/runtime's +# perf-build. +# +# ATOMICITY: the build/pack/upload steps that produce or move artifacts run +# continueOnError:false, so any failure sinks the build to Failed and the +# MissingBuildsTrigger Function skips indexing that SHA. The only exceptions are +# the always()-guarded best-effort log-publishing steps (continueOnError:true), +# which must never themselves fail an otherwise-good build. +# +# BCS output layout: +# builds/aspnetcore/buildArtifacts/{sha}/{configKey}/buildInfo.json +# builds/aspnetcore/buildArtifacts/{sha}/{configKey}/{archiveFile} +# + +parameters: +# Per-config enablement booleans (default true). Surfaced as pipeline parameters +# so they appear in the build's templateParameters for the dotnet-performance-infra +# MissingBuildsTrigger PerConfiguration indexer to count expected configs. In v1's +# eager per-commit mode all five are true; they are forward-compat for a future +# Function that queues a subset per config. +- name: aspnetcore_x64_linux + displayName: 'Build aspnetcore_x64_linux' + type: boolean + default: true +- name: aspnetcore_arm64_linux + displayName: 'Build aspnetcore_arm64_linux' + type: boolean + default: true +- name: aspnetcore_x64_windows + displayName: 'Build aspnetcore_x64_windows' + type: boolean + default: true +- name: aspnetcore_arm64_windows + displayName: 'Build aspnetcore_arm64_windows' + type: boolean + default: true +- name: aspnetcore_x86_windows + displayName: 'Build aspnetcore_x86_windows' + type: boolean + default: true + +# Self (performance) pushes must not fire this pipeline; only the aspnetcore +# resource trigger (below) does. +trigger: none +pr: none + +variables: +- name: _TeamName + value: AspNetCore +# Lean build args mirrored from aspnetcore ci-public.yml. Deliberately omit +# signing / installers / helix / publishing and any internal runtime download +# (public feeds only) -- none contribute to the per-RID runtime pack. +- name: _BuildArgs + value: /p:TeamName=$(_TeamName) + /p:OfficialBuildId=$(Build.BuildNumber) + /p:SkipTestBuild=true +- template: /eng/common/templates/variables/pool-providers.yml + +resources: + repositories: + # aspnetcore AzDO mirror. The CI trigger on main is what fires this pipeline + # per aspnetcore commit; the build jobs check it out at the triggering commit, + # and $(resources.repositories.aspnetcore.version) is the BCS {sha}. + - repository: aspnetcore + type: git + name: internal/dotnet-aspnetcore + ref: refs/heads/main + trigger: + batch: false + branches: + include: + - main + +stages: + +# ============================================================================ +# RegisterBuild -- writes a per-configKey buildInfo.json marker to BCS at +# $web/builds/aspnetcore/buildArtifacts/{sha}/{configKey}/buildInfo.json +# Runs in parallel with Build (dependsOn: []). Gated to internal + +# (ResourceTrigger | Manual): ResourceTrigger is the steady-state per-commit +# flow, Manual covers seed/runbook queues. +# +# The aspnetcore commit this build corresponds to is NOT tagged on the run: the +# dotnet-performance-infra indexer reads it directly from the build's +# `resources.repositories.aspnetcore.version` via the runs API (intrinsic to the +# build carrying the aspnetcore repo resource). The BCS {sha} path override below +# still uses that same value. +# ============================================================================ +- ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'ResourceTrigger'), eq(variables['Build.Reason'], 'Manual'))) }}: + - stage: RegisterBuild + displayName: 'Register Build' + dependsOn: [] + jobs: + - template: /eng/pipelines/register-build-jobs.yml + parameters: + repoName: aspnetcore + sha: $(resources.repositories.aspnetcore.version) + buildType: + - ${{ if eq(parameters.aspnetcore_x64_linux, true) }}: + - aspnetcore_x64_linux + - ${{ if eq(parameters.aspnetcore_arm64_linux, true) }}: + - aspnetcore_arm64_linux + - ${{ if eq(parameters.aspnetcore_x64_windows, true) }}: + - aspnetcore_x64_windows + - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}: + - aspnetcore_arm64_windows + - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}: + - aspnetcore_x86_windows + +# Build checks out the internal-only aspnetcore resource on the internal DncEng +# pool, so it cannot succeed outside the internal project. Gate it with the same +# internal + (ResourceTrigger | Manual) expression as the sibling stages. This is +# a runtime `condition:` (not the `${{ if }}` the siblings use) so the stage is +# present-but-skipped in public/non-trigger contexts -- if all three stages were +# `${{ if }}`-removed the pipeline would compile to zero stages, which is invalid. +- stage: Build + displayName: Build + dependsOn: [] + condition: and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'ResourceTrigger'), eq(variables['Build.Reason'], 'Manual'))) + jobs: + - template: /eng/pipelines/aspnetcore-perf-build-jobs.yml + parameters: + aspnetcoreRepoAlias: aspnetcore + performanceRepoAlias: self + aspnetcore_x64_linux: ${{ parameters.aspnetcore_x64_linux }} + aspnetcore_arm64_linux: ${{ parameters.aspnetcore_arm64_linux }} + aspnetcore_x64_windows: ${{ parameters.aspnetcore_x64_windows }} + aspnetcore_arm64_windows: ${{ parameters.aspnetcore_arm64_windows }} + aspnetcore_x86_windows: ${{ parameters.aspnetcore_x86_windows }} + +# ============================================================================ +# UploadArtifacts -- downloads each Build-stage pipeline artifact and uploads it +# to $web/builds/aspnetcore/buildArtifacts/{sha}/{configKey}/{file}. Depends on +# Build + RegisterBuild; condition succeeded() so any failure skips indexing. +# Same internal + (ResourceTrigger | Manual) gate as RegisterBuild. +# ============================================================================ +- ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'ResourceTrigger'), eq(variables['Build.Reason'], 'Manual'))) }}: + - stage: UploadArtifacts + displayName: 'Upload Artifacts to BCS' + dependsOn: + - Build + - RegisterBuild + condition: succeeded() + jobs: + - template: /eng/pipelines/upload-build-artifacts-jobs.yml + parameters: + repoName: aspnetcore + sha: $(resources.repositories.aspnetcore.version) + buildType: + - ${{ if eq(parameters.aspnetcore_x64_linux, true) }}: + - aspnetcore_x64_linux + - ${{ if eq(parameters.aspnetcore_arm64_linux, true) }}: + - aspnetcore_arm64_linux + - ${{ if eq(parameters.aspnetcore_x64_windows, true) }}: + - aspnetcore_x64_windows + - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}: + - aspnetcore_arm64_windows + - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}: + - aspnetcore_x86_windows diff --git a/eng/pipelines/register-build-jobs.yml b/eng/pipelines/register-build-jobs.yml index 7b3aa53a1bd..e998dd3d9b9 100644 --- a/eng/pipelines/register-build-jobs.yml +++ b/eng/pipelines/register-build-jobs.yml @@ -10,6 +10,11 @@ parameters: # pipeline; callers from other tenants (e.g. dotnet/aspnetcore) pass # repoName explicitly. repoName: 'runtime' + # sha is forwarded to each register-build-job so the buildInfo.json marker + # lands under builds/{repoName}/buildArtifacts/{sha}/... Defaults to the self + # repo commit; resource-built tenants (e.g. aspnetcore) pass the resource's + # commit, e.g. sha: $(resources.repositories.aspnetcore.version). + sha: '$(Build.SourceVersion)' jobs: - ${{ each type in parameters.buildType }}: @@ -18,3 +23,4 @@ jobs: buildType: ${{ type }} buildId: $(Build.BuildId) repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} diff --git a/eng/pipelines/templates/register-build-job.yml b/eng/pipelines/templates/register-build-job.yml index 7a275f64f2b..d397cbe22f0 100644 --- a/eng/pipelines/templates/register-build-job.yml +++ b/eng/pipelines/templates/register-build-job.yml @@ -7,6 +7,16 @@ parameters: # callers from other tenants (e.g. dotnet/aspnetcore) pass repoName # explicitly. repoName: 'runtime' + # sha controls the {sha} segment of the BCS blob path + # (builds/{repoName}/buildArtifacts/{sha}/...). Defaults to the self repo's + # triggering commit ($(Build.SourceVersion)) for back-compat with the runtime + # perf-build pipeline (which is hosted in dotnet/runtime, so self == the repo + # being built). Callers where the built repo is a *resource* rather than self + # (e.g. the dotnet/aspnetcore perf-build hosted in dotnet/performance) must + # pass the resource's commit explicitly, e.g. + # sha: $(resources.repositories.aspnetcore.version), because Build.SourceVersion + # would otherwise resolve to the performance pipeline commit. + sha: '$(Build.SourceVersion)' jobs: - job: RegisterBuild_${{ parameters.buildType }} @@ -20,4 +30,4 @@ jobs: scriptType: 'pscore' scriptLocation: 'inlineScript' inlineScript: | - az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --data '{"buildId":"${{ parameters.buildId }}"}' --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ variables['Build.SourceVersion'] }}/${{ parameters.buildType }}/buildInfo.json" --overwrite true + az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --data '{"buildId":"${{ parameters.buildId }}"}' --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ parameters.sha }}/${{ parameters.buildType }}/buildInfo.json" --overwrite true diff --git a/eng/pipelines/templates/upload-build-artifacts-job.yml b/eng/pipelines/templates/upload-build-artifacts-job.yml index 5da08e5553d..fd81e2d74d3 100644 --- a/eng/pipelines/templates/upload-build-artifacts-job.yml +++ b/eng/pipelines/templates/upload-build-artifacts-job.yml @@ -8,6 +8,16 @@ parameters: # Callers building a different repo (e.g. dotnet/aspnetcore) pass repoName explicitly # so that the upload lands at a tenant-specific path the indexer/crank then read. repoName: 'runtime' + # sha controls the {sha} segment of the BCS blob path: + # builds/{repoName}/buildArtifacts/{sha}/{buildType}/{file} + # Defaults to the self repo's triggering commit ($(Build.SourceVersion)) for + # back-compat with the runtime perf-build pipeline (hosted in dotnet/runtime, + # so self == the repo being built). Callers where the built repo is a + # *resource* rather than self (e.g. dotnet/aspnetcore perf-build hosted in + # dotnet/performance) must pass the resource's commit explicitly, e.g. + # sha: $(resources.repositories.aspnetcore.version), because Build.SourceVersion + # would otherwise resolve to the performance pipeline commit. + sha: '$(Build.SourceVersion)' jobs: - ${{ each artifact in parameters.artifacts }}: @@ -20,7 +30,7 @@ jobs: displayName: Download Build Artifact ${{ artifact.artifactName }} artifact: ${{ artifact.artifactName }} - ${{ each fileName in artifact.files }}: - - script: echo "pvscmdupload/\$web/builds/${{ parameters.repoName }}/buildArtifacts/${{ variables['Build.SourceVersion'] }}/${{ parameters.buildType }}/${{ fileName }}" + - script: echo "pvscmdupload/\$web/builds/${{ parameters.repoName }}/buildArtifacts/${{ parameters.sha }}/${{ parameters.buildType }}/${{ fileName }}" displayName: 'Artifact upload path' - task: AzureCLI@2 displayName: 'Upload ${{ fileName }} file' @@ -29,4 +39,4 @@ jobs: scriptType: 'pscore' scriptLocation: 'inlineScript' inlineScript: | - az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --file "$(Pipeline.Workspace)/${{ artifact.artifactName }}/${{ fileName }}" --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ variables['Build.SourceVersion'] }}/${{ parameters.buildType }}/${{ fileName }}" --overwrite true + az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --file "$(Pipeline.Workspace)/${{ artifact.artifactName }}/${{ fileName }}" --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ parameters.sha }}/${{ parameters.buildType }}/${{ fileName }}" --overwrite true diff --git a/eng/pipelines/tools/stage-bcs-nupkg-aspnetcore.ps1 b/eng/pipelines/tools/stage-bcs-nupkg-aspnetcore.ps1 new file mode 100644 index 00000000000..f10675c94ad --- /dev/null +++ b/eng/pipelines/tools/stage-bcs-nupkg-aspnetcore.ps1 @@ -0,0 +1,135 @@ +<# +.SYNOPSIS + Stages the per-RID Microsoft.AspNetCore.App runtime-pack nupkg as the Build + Cache Service (BCS) artifact for the perf-build pipeline. + +.DESCRIPTION + The single, canonical "find the runtime-pack nupkg and stage it under the BCS + artifact name" recipe used by every job in aspnetcore-perf-build-jobs.yml + (Windows x64/x86/arm64, Linux x64, Linux arm64). Each build job invokes it as + an explicit `pwsh` step with the RID(s) it produced. + + The BCS stores the runtime-pack nupkg VERBATIM (no extract/repack). A nupkg is + a zip on every OS, so dotnet/crank's BuildCacheClient extracts it with the same + ZipFile path on Windows and Linux; the nupkg's own `runtimes/{rid}/...` layout + is the load-bearing contract consumed there. Storing the raw nupkg preserves the + full shipped artifact (crank filters at consume time) rather than a lossy + projection that cannot be re-derived for historical bisection. + + For each RID it: + 1. Locates artifacts/packages/Release/Shipping/Microsoft.AspNetCore.App.Runtime.{rid}.*.nupkg + (excluding the *.symbols.nupkg), asserting exactly one match. + 2. Copies it to $StagingRoot/{artifactName}/{artifactName}.nupkg, renaming to + the fixed, version-independent BCS artifact name crank resolves from a sha. + +.PARAMETER Rids + One or more .NET RIDs to stage (e.g. win-x64, win-x86, win-arm64, linux-x64, + linux-arm64). os/arch and the BCS naming are derived from each RID. + +.PARAMETER ShippingDir + Directory containing the runtime-pack nupkgs. Defaults to + $(Build.SourcesDirectory)/artifacts/packages/Release/Shipping so the pipeline + can call this with no path arguments; override it for local runs. + +.PARAMETER StagingRoot + Directory under which the per-config artifact staging folders are created. + Defaults to $(Build.ArtifactStagingDirectory)/bcs. + +.EXAMPLE + # In the pipeline (ShippingDir/StagingRoot resolved from BUILD_* env vars): + ./stage-bcs-nupkg-aspnetcore.ps1 -Rids win-x64,win-x86,win-arm64 + +.EXAMPLE + # Local run -- the BUILD_* env vars are not set off-agent, so pass the + # directories explicitly: + ./stage-bcs-nupkg-aspnetcore.ps1 -Rids linux-x64 ` + -ShippingDir ./artifacts/packages/Release/Shipping ` + -StagingRoot ./artifacts/bcs +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string[]]$Rids, + + # Default to the BUILD_* agent paths in the pipeline; fall back to the current + # directory off-agent (those env vars are unset locally, and Join-Path $null + # throws during parameter binding), keeping the documented local-override flow. + [string]$ShippingDir = (Join-Path ($env:BUILD_SOURCESDIRECTORY ? $env:BUILD_SOURCESDIRECTORY : (Get-Location).Path) 'artifacts/packages/Release/Shipping'), + + [string]$StagingRoot = (Join-Path ($env:BUILD_ARTIFACTSTAGINGDIRECTORY ? $env:BUILD_ARTIFACTSTAGINGDIRECTORY : (Get-Location).Path) 'bcs') +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +function Get-RidParts { + param([string]$Rid) + + $dash = $Rid.IndexOf('-') + if ($dash -lt 1) { + throw "RID '$Rid' is not in the expected '-' form." + } + + $osToken = $Rid.Substring(0, $dash) + $arch = $Rid.Substring($dash + 1) + $os = switch ($osToken) { + 'win' { 'windows' } + 'linux' { 'linux' } + 'osx' { 'osx' } + default { throw "Unsupported RID os token '$osToken' in RID '$Rid'." } + } + + return [pscustomobject]@{ Os = $os; Arch = $arch } +} + +if (-not (Test-Path $ShippingDir)) { + throw "Shipping directory '$ShippingDir' does not exist." +} + +New-Item -ItemType Directory -Force -Path $StagingRoot | Out-Null + +foreach ($rid in $Rids) { + $parts = Get-RidParts -Rid $rid + $os = $parts.Os + $arch = $parts.Arch + $configKey = "aspnetcore_${arch}_${os}" + $artifactName = "BuildArtifacts_${os}_${arch}_Release_aspnetcore" + + Write-Host '' + Write-Host "=== Staging $configKey (rid=$rid) ===" + + $pattern = "Microsoft.AspNetCore.App.Runtime.$rid.*.nupkg" + $nupkgMatches = @(Get-ChildItem -Path $ShippingDir -Filter $pattern -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notmatch '\.symbols\.nupkg$' } | + Sort-Object Name) + if ($nupkgMatches.Count -eq 0) { + throw "Could not find runtime pack nupkg matching '$pattern' under '$ShippingDir'." + } + if ($nupkgMatches.Count -gt 1) { + # A clean from-source build produces exactly one non-symbols runtime pack + # per RID. More than one means a stale/duplicate version is lingering in + # the Shipping dir; pick-first would silently stage the lexicographically + # smallest (likely wrong) version, so fail loudly instead. + $names = ($nupkgMatches | ForEach-Object { $_.Name }) -join ', ' + throw "Expected exactly one runtime pack nupkg matching '$pattern' under '$ShippingDir', found $($nupkgMatches.Count): $names." + } + $nupkg = $nupkgMatches[0] + Write-Host "Found nupkg: $($nupkg.FullName)" + + # Stage the nupkg verbatim under the fixed, version-independent BCS artifact + # name. The real nupkg filename embeds the build version (e.g. + # ...win-x64.10.0.0-dev.nupkg), which crank cannot predict from a commit sha; + # crank resolves the blob as .../{configKey}/{artifactFile} with a fixed + # artifactFile, so we rename to the predictable name here. + $stageDir = Join-Path $StagingRoot $artifactName + if (Test-Path $stageDir) { + Remove-Item -LiteralPath $stageDir -Recurse -Force + } + New-Item -ItemType Directory -Force -Path $stageDir | Out-Null + + $stagedPath = Join-Path $stageDir "$artifactName.nupkg" + Copy-Item -LiteralPath $nupkg.FullName -Destination $stagedPath -Force + + $size = (Get-Item $stagedPath).Length + Write-Host "Staged nupkg: $stagedPath ($size bytes)" +} diff --git a/eng/pipelines/upload-build-artifacts-jobs.yml b/eng/pipelines/upload-build-artifacts-jobs.yml index b917db9457a..27899bbc2db 100644 --- a/eng/pipelines/upload-build-artifacts-jobs.yml +++ b/eng/pipelines/upload-build-artifacts-jobs.yml @@ -7,6 +7,14 @@ parameters: # back-compat with the existing dotnet/runtime perf-build pipeline; callers from # other tenants (e.g. dotnet/aspnetcore) pass repoName explicitly. repoName: 'runtime' + # sha controls the {sha} segment of the BCS blob path. Defaults to the self + # repo commit ($(Build.SourceVersion)) for back-compat with the runtime + # perf-build pipeline. Resource-built tenants (e.g. aspnetcore, whose source + # is a repository resource rather than self) pass the resource's commit, e.g. + # sha: $(resources.repositories.aspnetcore.version). All config branches below + # pass sha through explicitly; with no caller override it resolves to this + # $(Build.SourceVersion) default, so runtime behavior is unchanged. + sha: '$(Build.SourceVersion)' jobs: - ${{ if containsValue(parameters.buildType, 'coreclr_arm64_linux') }}: @@ -15,6 +23,7 @@ jobs: buildType: 'coreclr_arm64_linux' dependencyJobName: build_linux_arm64_release_coreclr repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_arm64_Release_coreclr' files: [ 'BuildArtifacts_linux_arm64_Release_coreclr.tar.gz' ] @@ -25,6 +34,7 @@ jobs: buildType: 'coreclr_arm64_windows' dependencyJobName: build_windows_arm64_release_coreclr repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_arm64_Release_coreclr' files: [ 'BuildArtifacts_windows_arm64_Release_coreclr.zip' ] @@ -35,6 +45,7 @@ jobs: buildType: 'coreclr_x64_linux' dependencyJobName: build_linux_x64_release_coreclr repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_x64_Release_coreclr' files: [ 'BuildArtifacts_linux_x64_Release_coreclr.tar.gz' ] @@ -45,6 +56,7 @@ jobs: buildType: 'coreclr_x64_windows' dependencyJobName: build_windows_x64_release_coreclr repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_x64_Release_coreclr' files: [ 'BuildArtifacts_windows_x64_Release_coreclr.zip' ] @@ -55,6 +67,7 @@ jobs: buildType: 'coreclr_x86_windows' dependencyJobName: build_windows_x86_release_coreclr repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_x86_Release_coreclr' files: [ 'BuildArtifacts_windows_x86_Release_coreclr.zip' ] @@ -65,6 +78,7 @@ jobs: buildType: 'coreclr_arm64_android' dependencyJobName: build_android_arm64_release_AndroidCoreCLR repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'AndroidHelloWorldArm64CoreCLR' files: [ 'AndroidHelloWorldArm64CoreCLR.tar.gz' ] @@ -75,6 +89,7 @@ jobs: buildType: 'wasm' dependencyJobName: build_browser_wasm_linux_Release_wasm repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BrowserWasm' files: [ 'BrowserWasm.tar.gz' ] @@ -85,6 +100,7 @@ jobs: buildType: 'monoAot_arm64_linux' dependencyJobName: build_linux_arm64_release_AOT repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'LinuxMonoAOTarm64' files: [ 'LinuxMonoAOTarm64.tar.gz' ] @@ -95,6 +111,7 @@ jobs: buildType: 'monoAot_x64_linux' dependencyJobName: build_linux_x64_release_AOT repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'LinuxMonoAOTx64' files: [ 'LinuxMonoAOTx64.tar.gz' ] @@ -105,6 +122,7 @@ jobs: buildType: 'mono_x64_linux' dependencyJobName: build_linux_x64_release_mono repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_x64_Release_mono' files: [ 'BuildArtifacts_linux_x64_Release_mono.tar.gz' ] @@ -115,6 +133,7 @@ jobs: buildType: 'mono_arm64_linux' dependencyJobName: build_linux_arm64_release_mono repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_arm64_Release_mono' files: [ 'BuildArtifacts_linux_arm64_Release_mono.tar.gz' ] @@ -125,6 +144,7 @@ jobs: buildType: 'nativeAot_arm64_ios' dependencyJobName: build_ios_arm64_release_iOSNativeAOT repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'iOSSampleAppNativeAOTNoSymbols' files: [ 'iOSSampleAppNativeAOTNoSymbols.zip' ] @@ -135,6 +155,7 @@ jobs: buildType: 'coreclr_arm64_ios' dependencyJobName: build_ios_arm64_release_iOSCoreCLR repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'iOSSampleAppCoreCLRInterpreterNoSymbols' files: [ 'iOSSampleAppCoreCLRInterpreterNoSymbols.zip' ] @@ -147,6 +168,7 @@ jobs: buildType: 'wasm_coreclr' dependencyJobName: build_browser_wasm_linux_Release_wasm_coreclr repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BrowserWasmCoreCLR' files: [ 'BrowserWasmCoreCLR.tar.gz' ] @@ -157,6 +179,7 @@ jobs: buildType: 'coreclr_r2r_interpreter' dependencyJobName: build_linux_x64_release_coreclr_r2r_interpreter repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_x64_Release_coreclr_r2r_interpreter' files: [ 'BuildArtifacts_linux_x64_Release_coreclr_r2r_interpreter.tar.gz' ] @@ -174,9 +197,10 @@ jobs: buildType: 'aspnetcore_x64_linux' dependencyJobName: Linux_x64_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_x64_Release_aspnetcore' - files: [ 'BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz' ] + files: [ 'BuildArtifacts_linux_x64_Release_aspnetcore.nupkg' ] - ${{ if containsValue(parameters.buildType, 'aspnetcore_arm64_linux') }}: - template: /eng/pipelines/templates/upload-build-artifacts-job.yml@${{ parameters.performanceRepoAlias }} @@ -184,36 +208,40 @@ jobs: buildType: 'aspnetcore_arm64_linux' dependencyJobName: Linux_arm64_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_arm64_Release_aspnetcore' - files: [ 'BuildArtifacts_linux_arm64_Release_aspnetcore.tar.gz' ] + files: [ 'BuildArtifacts_linux_arm64_Release_aspnetcore.nupkg' ] - ${{ if containsValue(parameters.buildType, 'aspnetcore_x64_windows') }}: - template: /eng/pipelines/templates/upload-build-artifacts-job.yml@${{ parameters.performanceRepoAlias }} parameters: buildType: 'aspnetcore_x64_windows' - dependencyJobName: Windows_build + dependencyJobName: Windows_x64_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_x64_Release_aspnetcore' - files: [ 'BuildArtifacts_windows_x64_Release_aspnetcore.zip' ] + files: [ 'BuildArtifacts_windows_x64_Release_aspnetcore.nupkg' ] - ${{ if containsValue(parameters.buildType, 'aspnetcore_arm64_windows') }}: - template: /eng/pipelines/templates/upload-build-artifacts-job.yml@${{ parameters.performanceRepoAlias }} parameters: buildType: 'aspnetcore_arm64_windows' - dependencyJobName: Windows_build + dependencyJobName: Windows_arm64_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_arm64_Release_aspnetcore' - files: [ 'BuildArtifacts_windows_arm64_Release_aspnetcore.zip' ] + files: [ 'BuildArtifacts_windows_arm64_Release_aspnetcore.nupkg' ] - ${{ if containsValue(parameters.buildType, 'aspnetcore_x86_windows') }}: - template: /eng/pipelines/templates/upload-build-artifacts-job.yml@${{ parameters.performanceRepoAlias }} parameters: buildType: 'aspnetcore_x86_windows' - dependencyJobName: Windows_build + dependencyJobName: Windows_x86_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_x86_Release_aspnetcore' - files: [ 'BuildArtifacts_windows_x86_Release_aspnetcore.zip' ] + files: [ 'BuildArtifacts_windows_x86_Release_aspnetcore.nupkg' ]