From 1f0faf3bd26e164d572f3e9a943432cf2725c5ed Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 15:44:15 -0700 Subject: [PATCH 1/8] Prototype host build tests on Helix Add a guarded public-pipeline path that prepares shared host toolchains and generates configurable duration-balanced Xamarin.Android.Build.Tests work items for Windows and Linux. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/HostBuildTestsHelix.proj | 46 ++ .../automation/azure-pipelines-public.yaml | 79 +++ .../HostBuildTestsHelix.Tests.ps1 | 135 +++++ .../HostBuildTestsHelix.psm1 | 468 ++++++++++++++++++ .../host-build-tests-helix/README.md | 24 + .../prepare-host-build-test-helix-payload.ps1 | 256 ++++++++++ .../run-host-build-tests-in-helix.yaml | 176 +++++++ 7 files changed, 1184 insertions(+) create mode 100644 build-tools/automation/HostBuildTestsHelix.proj create mode 100644 build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 create mode 100644 build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 create mode 100644 build-tools/automation/host-build-tests-helix/README.md create mode 100644 build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 create mode 100644 build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj new file mode 100644 index 00000000000..db68078dd58 --- /dev/null +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -0,0 +1,46 @@ + + + msbuild + true + false + true + tests/android/host-build-tests/ + Xamarin.Android.Build.Tests - Helix + 00:45:00 + + + + + + + $(HelixTestRunName) + + + + + + %(Identity) + + + + + + %(_HostBuildTestHelixWorkItem.PayloadDirectory) + %(_HostBuildTestHelixWorkItem.Command) + $(HostBuildTestsHelixWorkItemTimeout) + + + + + + + + + + + diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 44eae1ceb96..143ca87f891 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -31,6 +31,30 @@ parameters: values: - 'true' - 'false' +- name: enableHostBuildTestsHelixPrototype + displayName: Enable host build tests in Helix prototype + type: boolean + default: false +- name: hostBuildTestsHelixTargetMinutes + displayName: Host build tests Helix target minutes + type: number + default: 15 +- name: hostBuildTestsHelixDurationParallelism + displayName: Host build tests timing parallelism + type: number + default: 2.5 +- name: hostBuildTestsHelixFallbackTestDurationSeconds + displayName: Host build tests fallback seconds per test + type: number + default: 60 +- name: hostBuildTestsHelixTimingBuildId + displayName: Prior build ID containing host build test timings + type: string + default: '' +- name: hostBuildTestsHelixWorkItemTimeout + displayName: Host build tests Helix work item timeout + type: string + default: '00:45:00' # Repository resources resources: @@ -162,6 +186,61 @@ stages: nugetArtifactName: $(LinuxNuGetArtifactName) use1ESTemplate: false +# Guarded additive prototype. The existing Windows and Linux test jobs remain enabled. +- ${{ if eq(parameters.enableHostBuildTestsHelixPrototype, true) }}: + - stage: host_build_tests_helix + displayName: Host Build Tests Helix Prototype + dependsOn: + - mac_build + - linux_build + condition: and(succeeded(), ne(variables['SkipTestStages'], 'true')) + jobs: + - job: windows_host_build_tests_helix + displayName: Windows > Host Build Tests > Helix + pool: + name: $(NetCorePublicPoolName) + demands: + - ImageOverride -equals $(WindowsPoolImageNetCorePublic) + timeoutInMinutes: 300 + workspace: + clean: all + steps: + - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml + parameters: + platform: windows + helixQueue: Windows.Amd64.VS2026.Open + testFilter: '' + testRunTitle: Xamarin.Android.Build.Tests - Windows Helix Prototype + timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} + timingArtifactName: Host Build Test Timings - windows + targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }} + durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }} + fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }} + workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }} + + - job: linux_host_build_tests_helix + displayName: Linux > BuildTest > Helix + pool: + name: $(NetCorePublicPoolName) + demands: + - ImageOverride -equals $(LinuxPoolImageNetCorePublic) + timeoutInMinutes: 300 + workspace: + clean: all + steps: + - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml + parameters: + platform: linux + helixQueue: Ubuntu.2404.Amd64.Open + testFilter: 'Name = BuildTest' + testRunTitle: Xamarin.Android.Build.Tests - Linux BuildTest Helix Prototype + timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} + timingArtifactName: Host Build Test Timings - linux + targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }} + durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }} + fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }} + workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }} + # Package Tests Stage - ${{ if ne(variables.SkipTestStages, 'true') }}: - template: /build-tools/automation/yaml-templates/stage-package-tests.yaml diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 new file mode 100644 index 00000000000..3d6bb23cf40 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -0,0 +1,135 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'HostBuildTestsHelix.psm1') -Force + +function Assert-Equal +{ + param ( + [Parameter(Mandatory)] + [object] $Expected, + + [Parameter(Mandatory)] + [object] $Actual, + + [Parameter(Mandatory)] + [string] $Message + ) + + if ($Expected -ne $Actual) { + throw "$Message Expected '$Expected', got '$Actual'." + } +} + +$testRoot = Join-Path $PSScriptRoot '.test-output' +Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore +New-Item -ItemType Directory -Force -Path $testRoot | Out-Null + +try { + $emptyHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $fallbackPlan = New-DurationBalancedWorkItems ` + -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` + -TimingHistory $emptyHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 ` + -FallbackTestDurationSeconds 30 + + Assert-Equal 'count-fallback' $fallbackPlan.DurationMode 'No-history mode should be explicit.' + Assert-Equal 3 $fallbackPlan.WorkItems.Count 'Equal fallback durations should create three full bins.' + foreach ($workItem in $fallbackPlan.WorkItems) { + Assert-Equal 2 $workItem.Tests.Count 'Each fallback bin should contain two tests.' + Assert-Equal 60000 $workItem.EstimatedDurationMs 'Each fallback bin should target one minute.' + } + + $history = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $history.Add('Oversized', 720000) + $history.Add('Eight', 480000) + $history.Add('Six', 360000) + $history.Add('Four', 240000) + $historicalPlan = New-DurationBalancedWorkItems ` + -TestNames @('Four', 'Six', 'Eight', 'Oversized') ` + -TimingHistory $history ` + -TargetMinutes 10 ` + -DurationParallelism 1 + + Assert-Equal 'historical' $historicalPlan.DurationMode 'Complete history should be reported.' + Assert-Equal 3 $historicalPlan.WorkItems.Count 'Best-fit packing should create three bins.' + $oversized = @($historicalPlan.WorkItems | Where-Object Oversized) + Assert-Equal 1 $oversized.Count 'Only the over-target test should be oversized.' + Assert-Equal 1 $oversized[0].Tests.Count 'An over-target test must remain isolated.' + + $partialHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $partialHistory.Add('KnownOne', 10000) + $partialHistory.Add('KnownTwo', 30000) + $partialPlan = New-DurationBalancedWorkItems ` + -TestNames @('KnownOne', 'KnownTwo', 'NewTest') ` + -TimingHistory $partialHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 + + Assert-Equal 'partial-history' $partialPlan.DurationMode 'Missing tests should be reported as partial history.' + Assert-Equal 20000 $partialPlan.MissingTestDurationMs 'Missing tests should use the selected-history average.' + + $zeroTimingPath = Join-Path $testRoot 'zero-timing.xml' + Set-Content -LiteralPath $zeroTimingPath -Value '' + $zeroTimingHistory = Get-HostBuildTestTimingHistory -Path $zeroTimingPath + Assert-Equal 1 $zeroTimingHistory['FastTest'] 'Sub-millisecond TRX timings should retain a minimal positive load.' + + $specialNames = @("Namespace.Test('value')", 'Namespace.Path(C:\repo\file)') + $runSettingsPath = Join-Path $testRoot 'special.runsettings' + Write-NUnitRunSettings -TestNames $specialNames -Path $runSettingsPath + $roundTrippedNames = Get-NUnitRunSettingsTestNames -Path $runSettingsPath + Assert-Equal ($specialNames -join '|') ($roundTrippedNames -join '|') 'Runsettings names should round trip.' + + $secondFallbackPlan = New-DurationBalancedWorkItems ` + -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` + -TimingHistory $emptyHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 ` + -FallbackTestDurationSeconds 30 + Assert-Equal ` + ($fallbackPlan | ConvertTo-Json -Depth 8 -Compress) ` + ($secondFallbackPlan | ConvertTo-Json -Depth 8 -Compress) ` + 'Work-item generation should be deterministic.' + + $payloadDirectory = Join-Path $testRoot 'payloads' + $propsPath = Join-Path $testRoot 'work-items.props' + Write-HostBuildTestWorkItemPayloads ` + -Platform windows ` + -Plan $fallbackPlan ` + -WorkItemsDirectory $payloadDirectory ` + -WorkItemsPropsPath $propsPath ` + -TestAssemblyRelativePath 'bin\TestRelease\net10.0\Xamarin.Android.Build.Tests.dll' ` + -NUnitWorkers '-1' ` + -Configuration Release + + [xml] $props = Get-Content -LiteralPath $propsPath -Raw + Assert-Equal 3 @($props.Project.ItemGroup._HostBuildTestHelixWorkItem).Count 'Props should contain every generated work item.' + foreach ($item in @($props.Project.ItemGroup._HostBuildTestHelixWorkItem)) { + if (-not (Test-Path -LiteralPath $item.PayloadDirectory -PathType Container)) { + throw "Payload directory '$($item.PayloadDirectory)' was not created." + } + $command = Get-Content -LiteralPath (Join-Path $item.PayloadDirectory 'run-host-tests.cmd') -Raw + if ($command.Contains('__CONFIGURATION__') -or -not $command.Contains('bin\Release\dotnet')) { + throw 'The generated command did not apply the requested build configuration.' + } + } + + $linuxPayloadDirectory = Join-Path $testRoot 'linux-payloads' + $linuxPropsPath = Join-Path $testRoot 'linux-work-items.props' + Write-HostBuildTestWorkItemPayloads ` + -Platform linux ` + -Plan $fallbackPlan ` + -WorkItemsDirectory $linuxPayloadDirectory ` + -WorkItemsPropsPath $linuxPropsPath ` + -TestAssemblyRelativePath 'bin/TestRelease/net10.0/Xamarin.Android.Build.Tests.dll' ` + -NUnitWorkers '-1' ` + -Configuration Release + [xml] $linuxProps = Get-Content -LiteralPath $linuxPropsPath -Raw + foreach ($item in @($linuxProps.Project.ItemGroup._HostBuildTestHelixWorkItem)) { + Assert-Equal 'bash run-host-tests.sh' ([string] $item.Command) 'Linux work items should explicitly use bash.' + } + + Write-Host 'HostBuildTestsHelix tests passed.' +} finally { + Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore +} diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 new file mode 100644 index 00000000000..f51626c48b9 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -0,0 +1,468 @@ +Set-StrictMode -Version Latest + +function Get-NUnitRunSettingsTestNames +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + [xml] $runSettings = Get-Content -LiteralPath $Path -Raw + $where = [string] $runSettings.RunSettings.NUnit.Where + $prefix = "test == '" + $names = [System.Collections.Generic.List[string]]::new() + $offset = 0 + + while ($offset -lt $where.Length) { + $start = $where.IndexOf($prefix, $offset, [StringComparison]::Ordinal) + if ($start -lt 0) { + break + } + + $index = $start + $prefix.Length + $name = [Text.StringBuilder]::new() + $closed = $false + while ($index -lt $where.Length) { + $character = $where[$index] + if ($character -eq '\') { + if ($index + 1 -ge $where.Length) { + throw "Invalid escaped test name in '$Path'." + } + $index++ + [void] $name.Append($where[$index]) + } elseif ($character -eq "'") { + $closed = $true + $index++ + break + } else { + [void] $name.Append($character) + } + $index++ + } + + if (-not $closed) { + throw "Unterminated test name in '$Path'." + } + + $names.Add($name.ToString()) + $offset = $index + } + + if ($names.Count -eq 0) { + throw "No tests were found in '$Path'." + } + + return $names.ToArray() +} + +function Get-HostBuildTestTimingHistory +{ + param ( + [string] $Path + ) + + $durations = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + if ([string]::IsNullOrWhiteSpace($Path)) { + return $durations + } + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Timing history '$Path' does not exist." + } + + [xml] $balance = Get-Content -LiteralPath $Path -Raw + foreach ($test in @($balance.tests.test)) { + $name = [string] $test.name + $duration = 0L + if ([string]::IsNullOrWhiteSpace($name) -or + -not [long]::TryParse([string] $test.duration, [ref] $duration) -or + $duration -lt 0) { + throw "Timing history '$Path' contains an invalid test entry." + } + if (-not $durations.TryAdd($name, [Math]::Max(1, $duration))) { + throw "Timing history '$Path' contains duplicate test '$name'." + } + } + + return $durations +} + +function New-DurationBalancedWorkItems +{ + param ( + [Parameter(Mandatory)] + [string[]] $TestNames, + + [Parameter(Mandatory)] + [System.Collections.Generic.IDictionary[string,long]] $TimingHistory, + + [Parameter(Mandatory)] + [ValidateRange(1, 1440)] + [int] $TargetMinutes, + + [Parameter(Mandatory)] + [ValidateRange(0.1, 128)] + [double] $DurationParallelism, + + [ValidateRange(1, 86400)] + [int] $FallbackTestDurationSeconds = 60 + ) + + $uniqueNames = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($name in $TestNames) { + if ([string]::IsNullOrWhiteSpace($name)) { + throw 'Test names cannot be empty.' + } + if (-not $uniqueNames.Add($name)) { + throw "Duplicate test name '$name'." + } + } + if ($uniqueNames.Count -eq 0) { + throw 'At least one test is required.' + } + + $knownDurations = [System.Collections.Generic.List[long]]::new() + foreach ($name in $uniqueNames) { + $duration = 0L + if ($TimingHistory.TryGetValue($name, [ref] $duration)) { + $knownDurations.Add($duration) + } + } + + if ($knownDurations.Count -eq 0) { + $missingDuration = [long] $FallbackTestDurationSeconds * 1000 + $durationMode = 'count-fallback' + } else { + $missingDuration = [long] [Math]::Round( + ($knownDurations | Measure-Object -Average).Average, + [MidpointRounding]::AwayFromZero + ) + $durationMode = if ($knownDurations.Count -eq $uniqueNames.Count) { 'historical' } else { 'partial-history' } + } + + $tests = [System.Collections.Generic.List[object]]::new() + foreach ($name in $uniqueNames) { + $duration = 0L + $isEstimated = -not $TimingHistory.TryGetValue($name, [ref] $duration) + if ($isEstimated) { + $duration = $missingDuration + } + $tests.Add([pscustomobject] @{ + Name = $name + DurationMs = $duration + IsEstimated = $isEstimated + }) + } + + $comparison = [System.Comparison[object]] { + param ($left, $right) + $durationComparison = $right.DurationMs.CompareTo($left.DurationMs) + if ($durationComparison -ne 0) { + return $durationComparison + } + return [StringComparer]::Ordinal.Compare($left.Name, $right.Name) + } + $tests.Sort($comparison) + + $targetLoadMs = [long] [Math]::Round( + $TargetMinutes * 60000 * $DurationParallelism, + [MidpointRounding]::AwayFromZero + ) + $bins = [System.Collections.Generic.List[object]]::new() + + foreach ($test in $tests) { + $selectedBin = $null + if ($test.DurationMs -le $targetLoadMs) { + foreach ($bin in $bins) { + if ($bin.Oversized -or $bin.DurationMs + $test.DurationMs -gt $targetLoadMs) { + continue + } + if ($null -eq $selectedBin -or + $bin.DurationMs -gt $selectedBin.DurationMs -or + ($bin.DurationMs -eq $selectedBin.DurationMs -and $bin.Id -lt $selectedBin.Id)) { + $selectedBin = $bin + } + } + } + + if ($null -eq $selectedBin) { + $selectedBin = [pscustomobject] @{ + Id = $bins.Count + 1 + DurationMs = 0L + Oversized = $test.DurationMs -gt $targetLoadMs + Tests = [System.Collections.Generic.List[object]]::new() + } + $bins.Add($selectedBin) + } + + $selectedBin.Tests.Add($test) + $selectedBin.DurationMs += $test.DurationMs + } + + $workItems = [System.Collections.Generic.List[object]]::new() + foreach ($bin in $bins) { + $sortedNames = [System.Collections.Generic.List[string]]::new() + foreach ($test in $bin.Tests) { + $sortedNames.Add($test.Name) + } + $sortedNames.Sort([StringComparer]::Ordinal) + + $workItems.Add([pscustomobject] @{ + Id = $bin.Id + DurationMs = $bin.DurationMs + EstimatedDurationMs = [long] [Math]::Ceiling($bin.DurationMs / $DurationParallelism) + Oversized = $bin.Oversized + Tests = $sortedNames.ToArray() + }) + } + + return [pscustomobject] @{ + DurationMode = $durationMode + KnownTestCount = $knownDurations.Count + EstimatedTestCount = $uniqueNames.Count - $knownDurations.Count + MissingTestDurationMs = $missingDuration + TargetLoadMs = $targetLoadMs + TargetMinutes = $TargetMinutes + DurationParallelism = $DurationParallelism + TotalDurationMs = ($tests | Measure-Object -Property DurationMs -Sum).Sum + WorkItems = $workItems.ToArray() + } +} + +function Write-NUnitRunSettings +{ + param ( + [Parameter(Mandatory)] + [string[]] $TestNames, + + [Parameter(Mandatory)] + [string] $Path + ) + + $settings = [Xml.XmlWriterSettings]::new() + $settings.Indent = $true + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $writer = [Xml.XmlWriter]::Create($Path, $settings) + try { + $filters = foreach ($name in $TestNames) { + $escaped = $name.Replace('\', '\\').Replace("'", "\'") + "test == '$escaped'" + } + $writer.WriteStartElement('RunSettings') + $writer.WriteStartElement('NUnit') + $writer.WriteElementString('Where', ($filters -join ' or ')) + $writer.WriteEndElement() + $writer.WriteEndElement() + } finally { + $writer.Dispose() + } +} + +function Write-HostBuildTestCommand +{ + param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $NUnitWorkers, + + [Parameter(Mandatory)] + [string] $Configuration, + + [Parameter(Mandatory)] + [string] $Path + ) + + if ($Platform -eq 'windows') { + $assemblyPath = $TestAssemblyRelativePath.Replace('/', '\') + $command = @' +@echo off +setlocal +if "%HELIX_CORRELATION_PAYLOAD%"=="" exit /b 2 +if "%HELIX_WORKITEM_ROOT%"=="" exit /b 2 +if "%HELIX_WORKITEM_UPLOAD_ROOT%"=="" exit /b 2 +set "REPO=%HELIX_CORRELATION_PAYLOAD%\repo" +set "DOTNET_ROOT=%REPO%\bin\__CONFIGURATION__\dotnet" +set "ANDROID_HOME=%HELIX_CORRELATION_PAYLOAD%\android-toolchain\sdk" +set "ANDROID_SDK_ROOT=%ANDROID_HOME%" +set "JAVA_HOME=%HELIX_CORRELATION_PAYLOAD%\jdk" +set "NUGET_PACKAGES=%HELIX_CORRELATION_PAYLOAD%\nuget-packages" +set "GRADLE_USER_HOME=%HELIX_CORRELATION_PAYLOAD%\gradle" +set "DOTNET_CLI_HOME=%HELIX_WORKITEM_ROOT%\dotnet-home" +set "TEMP=%HELIX_WORKITEM_ROOT%\temp" +set "TMP=%TEMP%" +set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" +set "CONFIGURATION=__CONFIGURATION__" +set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" +set "DOTNET_CLI_TELEMETRY_OPTOUT=1" +set "DOTNET_NOLOGO=1" +set "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" +set "DOTNET_GENERATE_ASPNET_CERTIFICATE=false" +set "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=true" +set "DOTNET_SDK_VULNERABILITY_CHECK_DISABLE=true" +set "DOTNET_SYSTEM_NET_SECURITY_NOREVOCATIONCHECKBYDEFAULT=true" +set "NuGetAudit=false" +set "DOTNET_DbgEnableMiniDump=1" +set "DOTNET_DbgMiniDumpType=2" +set "DOTNET_DbgMiniDumpName=%HELIX_WORKITEM_UPLOAD_ROOT%\dumps\%%e.%%p.dmp" +if not exist "%DOTNET_ROOT%\dotnet.exe" exit /b 3 +if not exist "%REPO%\__TEST_ASSEMBLY__" exit /b 3 +mkdir "%HELIX_WORKITEM_UPLOAD_ROOT%\dumps" 2>nul +mkdir "%DOTNET_CLI_HOME%" 2>nul +mkdir "%TEMP%" 2>nul +copy /y "%~dp0slice.runsettings" "%HELIX_WORKITEM_UPLOAD_ROOT%\slice.runsettings" >nul +copy /y "%~dp0work-item.json" "%HELIX_WORKITEM_UPLOAD_ROOT%\work-item.json" >nul +pushd "%REPO%" +"%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" 2>&1 +set "testExitCode=%ERRORLEVEL%" +type "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" +"%DOTNET_ROOT%\dotnet.exe" build-server shutdown >nul 2>&1 +popd +exit /b %testExitCode% +'@ + $command = $command.Replace('__TEST_ASSEMBLY__', $assemblyPath) + $command = $command.Replace('__NUNIT_WORKERS__', $NUnitWorkers) + $command = $command.Replace('__CONFIGURATION__', $Configuration) + [IO.File]::WriteAllText($Path, $command, [Text.Encoding]::ASCII) + } else { + $assemblyPath = $TestAssemblyRelativePath.Replace('\', '/') + $command = @' +#!/usr/bin/env bash +set -euo pipefail +: "${HELIX_CORRELATION_PAYLOAD:?}" +: "${HELIX_WORKITEM_ROOT:?}" +: "${HELIX_WORKITEM_UPLOAD_ROOT:?}" +REPO="$HELIX_CORRELATION_PAYLOAD/repo" +export DOTNET_ROOT="$REPO/bin/__CONFIGURATION__/dotnet" +export ANDROID_HOME="$HELIX_CORRELATION_PAYLOAD/android-toolchain/sdk" +export ANDROID_SDK_ROOT="$ANDROID_HOME" +export JAVA_HOME="$HELIX_CORRELATION_PAYLOAD/jdk" +export NUGET_PACKAGES="$HELIX_CORRELATION_PAYLOAD/nuget-packages" +export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" +export DOTNET_CLI_HOME="$HELIX_WORKITEM_ROOT/dotnet-home" +export TMPDIR="$HELIX_WORKITEM_ROOT/temp" +export BUILD_STAGINGDIRECTORY="$HELIX_WORKITEM_UPLOAD_ROOT" +export CONFIGURATION=__CONFIGURATION__ +export PATH="$HELIX_CORRELATION_PAYLOAD/dotnet-tools:$DOTNET_ROOT:$ANDROID_HOME/platform-tools:$JAVA_HOME/bin:$PATH" +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_NOLOGO=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +export DOTNET_GENERATE_ASPNET_CERTIFICATE=false +export DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=true +export DOTNET_SDK_VULNERABILITY_CHECK_DISABLE=true +export DOTNET_SYSTEM_NET_SECURITY_NOREVOCATIONCHECKBYDEFAULT=true +export NuGetAudit=false +export DOTNET_DbgEnableMiniDump=1 +export DOTNET_DbgMiniDumpType=2 +export DOTNET_DbgMiniDumpName="$HELIX_WORKITEM_UPLOAD_ROOT/dumps/%e.%p.dmp" +test -x "$DOTNET_ROOT/dotnet" +test -f "$REPO/__TEST_ASSEMBLY__" +mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$TMPDIR" +cp "$HELIX_WORKITEM_ROOT/slice.runsettings" "$HELIX_WORKITEM_UPLOAD_ROOT/slice.runsettings" +cp "$HELIX_WORKITEM_ROOT/work-item.json" "$HELIX_WORKITEM_UPLOAD_ROOT/work-item.json" +cd "$REPO" +set +e +"$DOTNET_ROOT/dotnet" test "$REPO/__TEST_ASSEMBLY__" --settings "$HELIX_WORKITEM_ROOT/slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "$HELIX_WORKITEM_UPLOAD_ROOT" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ 2>&1 | tee "$HELIX_WORKITEM_UPLOAD_ROOT/console.log" +test_exit=${PIPESTATUS[0]} +set -e +"$DOTNET_ROOT/dotnet" build-server shutdown >/dev/null 2>&1 || true +exit "$test_exit" +'@ + $command = $command.Replace('__TEST_ASSEMBLY__', $assemblyPath) + $command = $command.Replace('__NUNIT_WORKERS__', $NUnitWorkers) + $command = $command.Replace('__CONFIGURATION__', $Configuration) + [IO.File]::WriteAllText($Path, $command, [Text.UTF8Encoding]::new($false)) + } +} + +function Write-HostBuildTestWorkItemPayloads +{ + param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [object] $Plan, + + [Parameter(Mandatory)] + [string] $WorkItemsDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsPropsPath, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $NUnitWorkers, + + [Parameter(Mandatory)] + [string] $Configuration + ) + + Remove-Item -LiteralPath $WorkItemsDirectory -Recurse -Force -ErrorAction Ignore + New-Item -ItemType Directory -Force -Path $WorkItemsDirectory | Out-Null + + $propsSettings = [Xml.XmlWriterSettings]::new() + $propsSettings.Indent = $true + $propsSettings.Encoding = [Text.UTF8Encoding]::new($false) + $propsWriter = [Xml.XmlWriter]::Create($WorkItemsPropsPath, $propsSettings) + try { + $propsWriter.WriteStartElement('Project') + $propsWriter.WriteStartElement('ItemGroup') + + foreach ($workItem in $Plan.WorkItems) { + $name = 'host-build-tests-{0}-{1:d3}' -f $Platform, $workItem.Id + $payloadDirectory = Join-Path $WorkItemsDirectory $name + New-Item -ItemType Directory -Force -Path $payloadDirectory | Out-Null + + Write-NUnitRunSettings -TestNames $workItem.Tests -Path (Join-Path $payloadDirectory 'slice.runsettings') + $commandName = if ($Platform -eq 'windows') { 'run-host-tests.cmd' } else { 'run-host-tests.sh' } + $helixCommand = if ($Platform -eq 'windows') { $commandName } else { "bash $commandName" } + Write-HostBuildTestCommand ` + -Platform $Platform ` + -TestAssemblyRelativePath $TestAssemblyRelativePath ` + -NUnitWorkers $NUnitWorkers ` + -Configuration $Configuration ` + -Path (Join-Path $payloadDirectory $commandName) + + [pscustomobject] @{ + name = $name + durationMode = $Plan.DurationMode + estimatedDurationMs = $workItem.EstimatedDurationMs + durationLoadMs = $workItem.DurationMs + oversized = $workItem.Oversized + targetMinutes = $Plan.TargetMinutes + durationParallelism = $Plan.DurationParallelism + testCount = $workItem.Tests.Count + tests = $workItem.Tests + } | ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath (Join-Path $payloadDirectory 'work-item.json') -Encoding utf8NoBOM + + $propsWriter.WriteStartElement('_HostBuildTestHelixWorkItem') + $propsWriter.WriteAttributeString('Include', $name) + $propsWriter.WriteElementString('PayloadDirectory', $payloadDirectory) + $propsWriter.WriteElementString('Command', $helixCommand) + $propsWriter.WriteElementString('EstimatedDurationMs', [string] $workItem.EstimatedDurationMs) + $propsWriter.WriteElementString('TestCount', [string] $workItem.Tests.Count) + $propsWriter.WriteEndElement() + } + + $propsWriter.WriteEndElement() + $propsWriter.WriteEndElement() + } finally { + $propsWriter.Dispose() + } +} + +Export-ModuleMember -Function ` + Get-NUnitRunSettingsTestNames, ` + Get-HostBuildTestTimingHistory, ` + New-DurationBalancedWorkItems, ` + Write-NUnitRunSettings, ` + Write-HostBuildTestWorkItemPayloads diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md new file mode 100644 index 00000000000..905b40f0cc6 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -0,0 +1,24 @@ +# Host build tests in Helix prototype + +This guarded prototype stages the already-prepared .NET SDK/workload, Android SDK, +JDK, NuGet/Gradle caches, selected repository inputs, and test assemblies into one +Helix correlation payload per host OS. Work item payloads contain only a runsettings +filter, a small command script, and generation metadata. To control payload size, the +source copy omits `src/Mono.Android`; host build tests consume that product through the +prepared workload packs instead. + +`prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's +`dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit +decreasing bin packing. The target is `15` estimated wall-clock minutes by default. +`durationParallelism` translates summed per-test elapsed time into expected wall time +while preserving the current NUnit worker setting. The public prototype defaults this +factor to `2.5`, uses `60` seconds per test for the explicit first-run fallback, and +allows `45` minutes before Helix terminates a work item. + +If no timing history is provided, generation explicitly reports `count-fallback` and +uses the configured fallback duration per test. It does not label count-only slicing +as duration-balanced. Each completed run publishes a new `balance.xml`; pass that +run's build ID through `hostBuildTestsHelixTimingBuildId` on the next queued build. + +The public pipeline keeps the existing Azure Pipelines jobs and adds this path only +when `enableHostBuildTestsHelixPrototype` is `true`. diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 new file mode 100644 index 00000000000..23c779f4764 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -0,0 +1,256 @@ +[CmdletBinding()] +param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [string] $RepositoryRoot, + + [Parameter(Mandatory)] + [string] $CorrelationPayloadDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsPropsPath, + + [Parameter(Mandatory)] + [string] $ResultsDirectory, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $TestSlicerPath, + + [string] $TestFilter = '', + + [string] $BalanceFile = '', + + [ValidateRange(1, 1440)] + [int] $TargetMinutes = 15, + + [ValidateRange(0.1, 128)] + [double] $DurationParallelism = 2.5, + + [ValidateRange(1, 86400)] + [int] $FallbackTestDurationSeconds = 60, + + [string] $NUnitWorkers = '-1', + + [string] $Configuration = 'Release', + + [Parameter(Mandatory)] + [string] $AndroidHome, + + [Parameter(Mandatory)] + [string] $JavaHome, + + [string] $NuGetPackages = '', + + [string] $DotNetToolsDirectory = '', + + [string] $GradleUserHome = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'HostBuildTestsHelix.psm1') -Force + +function ConvertTo-NativeRelativePath +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $separator = [string] [IO.Path]::DirectorySeparatorChar + return $Path.Replace('\', $separator).Replace('/', $separator) +} + +function Copy-PayloadDirectory +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [switch] $Optional + ) + + if (-not (Test-Path -LiteralPath $Source -PathType Container)) { + if ($Optional) { + Write-Warning "Optional payload directory '$Source' does not exist." + return + } + throw "Required payload directory '$Source' does not exist." + } + + Write-Host "Staging payload directory '$Source'." + New-Item -ItemType Directory -Force -Path $Destination | Out-Null + if ($IsWindows) { + & robocopy $Source $Destination /E /COPY:DAT /DCOPY:DAT /R:2 /W:1 /NFL /NDL /NJH /NJS /NP + if ($LASTEXITCODE -gt 7) { + throw "robocopy failed with exit code $LASTEXITCODE while copying '$Source'." + } + } else { + & cp -a "$Source/." "$Destination/" + if ($LASTEXITCODE -ne 0) { + throw "cp failed with exit code $LASTEXITCODE while copying '$Source'." + } + } +} + +$repositoryRootFullPath = [IO.Path]::GetFullPath($RepositoryRoot) +$correlationPayloadFullPath = [IO.Path]::GetFullPath($CorrelationPayloadDirectory) +$workItemsFullPath = [IO.Path]::GetFullPath($WorkItemsDirectory) +$workItemsPropsFullPath = [IO.Path]::GetFullPath($WorkItemsPropsPath) +$resultsFullPath = [IO.Path]::GetFullPath($ResultsDirectory) + +Remove-Item -LiteralPath $correlationPayloadFullPath -Recurse -Force -ErrorAction Ignore +Remove-Item -LiteralPath $workItemsFullPath -Recurse -Force -ErrorAction Ignore +New-Item -ItemType Directory -Force -Path $correlationPayloadFullPath, $workItemsFullPath, $resultsFullPath | Out-Null + +$payloadRepository = Join-Path $correlationPayloadFullPath 'repo' +New-Item -ItemType Directory -Force -Path $payloadRepository | Out-Null +Get-ChildItem -LiteralPath $repositoryRootFullPath -File | Copy-Item -Destination $payloadRepository + +$payloadDirectories = @( + @("bin\$Configuration", "bin\$Configuration", $false), + @("bin\Test$Configuration", "bin\Test$Configuration", $false), + @('build-tools', 'build-tools', $false), + @('eng', 'eng', $false), + @('external', 'external', $false), + @('.github', '.github', $true), + @('samples', 'samples', $true), + @('src-ThirdParty', 'src-ThirdParty', $true), + @('tests', 'tests', $false), + @('tools', 'tools', $true) +) +foreach ($directory in $payloadDirectories) { + Copy-PayloadDirectory ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $directory[0])) ` + -Destination (Join-Path $payloadRepository (ConvertTo-NativeRelativePath $directory[1])) ` + -Optional:([bool] $directory[2]) +} + +$sourceRoot = Join-Path $repositoryRootFullPath 'src' +foreach ($sourceDirectory in Get-ChildItem -LiteralPath $sourceRoot -Directory | Where-Object Name -ne 'Mono.Android') { + Copy-PayloadDirectory ` + -Source $sourceDirectory.FullName ` + -Destination (Join-Path (Join-Path $payloadRepository 'src') $sourceDirectory.Name) +} + +Copy-PayloadDirectory -Source $AndroidHome -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath 'android-toolchain/sdk')) +Copy-PayloadDirectory -Source $JavaHome -Destination (Join-Path $correlationPayloadFullPath 'jdk') + +if ([string]::IsNullOrWhiteSpace($NuGetPackages)) { + $NuGetPackages = Join-Path ([Environment]::GetFolderPath('UserProfile')) (ConvertTo-NativeRelativePath '.nuget/packages') +} +Copy-PayloadDirectory -Source $NuGetPackages -Destination (Join-Path $correlationPayloadFullPath 'nuget-packages') + +if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { + $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' +} +foreach ($gradleDirectory in @('caches', 'wrapper')) { + Copy-PayloadDirectory ` + -Source (Join-Path $GradleUserHome $gradleDirectory) ` + -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath "gradle/$gradleDirectory")) ` + -Optional +} + +if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { + $payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' + New-Item -ItemType Directory -Force -Path $payloadTools | Out-Null + $apkDiffShim = if ($Platform -eq 'windows') { 'apkdiff.exe' } else { 'apkdiff' } + $apkDiffShimPath = Join-Path $DotNetToolsDirectory $apkDiffShim + if (Test-Path -LiteralPath $apkDiffShimPath -PathType Leaf) { + Copy-Item -LiteralPath $apkDiffShimPath -Destination $payloadTools + } + Copy-PayloadDirectory ` + -Source (Join-Path $DotNetToolsDirectory (ConvertTo-NativeRelativePath '.store/apkdiff')) ` + -Destination (Join-Path $payloadTools (ConvertTo-NativeRelativePath '.store/apkdiff')) ` + -Optional +} + +$stagedTestAssembly = Join-Path $payloadRepository (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) +if (-not (Test-Path -LiteralPath $stagedTestAssembly -PathType Leaf)) { + throw "Test assembly '$stagedTestAssembly' was not staged." +} +if (-not (Test-Path -LiteralPath $TestSlicerPath -PathType Leaf)) { + throw "dotnet-test-slicer '$TestSlicerPath' does not exist." +} + +$discoveryRunSettings = Join-Path $workItemsFullPath 'all-tests.runsettings' +$sliceArguments = @( + 'slice', + "--test-assembly=$stagedTestAssembly", + '--slice-number=1', + '--total-slices=1', + "--outfile=$discoveryRunSettings" +) +if (-not [string]::IsNullOrWhiteSpace($TestFilter)) { + $sliceArguments += "--test-filter=$TestFilter" +} + +& $TestSlicerPath @sliceArguments +if ($LASTEXITCODE -ne 0) { + throw "dotnet-test-slicer discovery failed with exit code $LASTEXITCODE." +} + +$testNames = Get-NUnitRunSettingsTestNames -Path $discoveryRunSettings +$timingHistory = Get-HostBuildTestTimingHistory -Path $BalanceFile +$plan = New-DurationBalancedWorkItems ` + -TestNames $testNames ` + -TimingHistory $timingHistory ` + -TargetMinutes $TargetMinutes ` + -DurationParallelism $DurationParallelism ` + -FallbackTestDurationSeconds $FallbackTestDurationSeconds + +Write-HostBuildTestWorkItemPayloads ` + -Platform $Platform ` + -Plan $plan ` + -WorkItemsDirectory $workItemsFullPath ` + -WorkItemsPropsPath $workItemsPropsFullPath ` + -TestAssemblyRelativePath $TestAssemblyRelativePath ` + -NUnitWorkers $NUnitWorkers ` + -Configuration $Configuration + +$summaryPath = Join-Path $resultsFullPath 'work-item-generation.json' +[pscustomobject] @{ + platform = $Platform + testFilter = $TestFilter + durationMode = $plan.DurationMode + targetMinutes = $TargetMinutes + durationParallelism = $DurationParallelism + fallbackTestDurationSeconds = $FallbackTestDurationSeconds + testCount = $testNames.Count + knownTestCount = $plan.KnownTestCount + estimatedTestCount = $plan.EstimatedTestCount + workItemCount = $plan.WorkItems.Count + totalDurationLoadMs = $plan.TotalDurationMs + workItems = @($plan.WorkItems | ForEach-Object { + [pscustomobject] @{ + id = $_.Id + testCount = $_.Tests.Count + durationLoadMs = $_.DurationMs + estimatedDurationMs = $_.EstimatedDurationMs + oversized = $_.Oversized + } + }) +} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $summaryPath -Encoding utf8NoBOM + +Write-Host "Generated $($plan.WorkItems.Count) $Platform Helix work items for $($testNames.Count) tests." +Write-Host "Duration mode: $($plan.DurationMode); known timings: $($plan.KnownTestCount); estimated timings: $($plan.EstimatedTestCount)." +Write-Host "Target: $TargetMinutes minute(s); duration parallelism: $DurationParallelism." +if ($plan.DurationMode -eq 'count-fallback') { + Write-Warning "No matching timing history was available. Work items use an explicit count fallback of $FallbackTestDurationSeconds second(s) per test." +} + +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixCorrelationPayload]$correlationPayloadFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemsProps]$workItemsPropsFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemCount]$($plan.WorkItems.Count)" diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml new file mode 100644 index 00000000000..6132358d98e --- /dev/null +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -0,0 +1,176 @@ +parameters: + platform: '' + helixQueue: '' + testFilter: '' + testRunTitle: '' + timingBuildId: '' + timingArtifactName: '' + targetMinutes: 15 + durationParallelism: 2.5 + fallbackTestDurationSeconds: 60 + workItemTimeout: '00:45:00' + nunitWorkers: '-1' + +steps: +- template: /build-tools/automation/yaml-templates/setup-test-environment-public.yaml + parameters: + useAgentJdkPath: false + installTestSlicer: true + use1ESTemplate: false + +- template: /build-tools/automation/yaml-templates/cache-gradle.yaml + parameters: + xaSourcePath: $(System.DefaultWorkingDirectory) + +- task: DownloadPipelineArtifact@2 + displayName: Download Xamarin.Android.Build.Tests assets + inputs: + artifactName: $(TestAssembliesArtifactName) + downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) + +- ${{ if ne(parameters.timingBuildId, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download prior ${{ parameters.platform }} host-test timings + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + buildVersionToDownload: specific + pipelineId: ${{ parameters.timingBuildId }} + artifactName: ${{ parameters.timingArtifactName }} + targetPath: $(Pipeline.Workspace)/host-build-test-history-${{ parameters.platform }} + +- pwsh: | + $ErrorActionPreference = 'Stop' + $platform = '${{ parameters.platform }}' + $historyRoot = '$(Pipeline.Workspace)/host-build-test-history-${{ parameters.platform }}' + $balanceFile = '' + if ('${{ parameters.timingBuildId }}') { + $balanceFile = Get-ChildItem -LiteralPath $historyRoot -Recurse -Filter '*.xml' | + Select-Object -First 1 -ExpandProperty FullName + if (-not $balanceFile) { + throw "Build '${{ parameters.timingBuildId }}' did not provide timing XML in '$historyRoot'." + } + } + + $testSlicer = Join-Path '$(Agent.ToolsDirectory)' $(if ($platform -eq 'windows') { 'dotnet-test-slicer.exe' } else { 'dotnet-test-slicer' }) + $stagingRoot = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}' + $resultsDirectory = Join-Path $stagingRoot 'results' + $correlationPayload = Join-Path $stagingRoot 'correlation' + $workItemsDirectory = Join-Path $stagingRoot 'work-items' + $workItemsProps = Join-Path $stagingRoot 'work-items.props' + $adbName = if ($platform -eq 'windows') { 'adb.exe' } else { 'adb' } + $androidHome = @( + $env:ANDROID_HOME + $env:ANDROID_SDK_ROOT + $env:AndroidSdkDirectory + $(if ($env:USERPROFILE) { Join-Path $env:USERPROFILE 'android-toolchain\sdk' }) + $(if ($env:HOME) { Join-Path $env:HOME 'android-toolchain/sdk' }) + ) | Where-Object { $_ -and (Test-Path (Join-Path $_ "platform-tools/$adbName")) } | Select-Object -First 1 + if (-not $androidHome) { + throw 'Could not locate the prepared Android SDK.' + } + + & '$(System.DefaultWorkingDirectory)/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1' ` + -Platform $platform ` + -RepositoryRoot '$(System.DefaultWorkingDirectory)' ` + -CorrelationPayloadDirectory $correlationPayload ` + -WorkItemsDirectory $workItemsDirectory ` + -WorkItemsPropsPath $workItemsProps ` + -ResultsDirectory $resultsDirectory ` + -TestAssemblyRelativePath 'bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll' ` + -TestSlicerPath $testSlicer ` + -TestFilter '${{ parameters.testFilter }}' ` + -BalanceFile $balanceFile ` + -TargetMinutes ${{ parameters.targetMinutes }} ` + -DurationParallelism ${{ parameters.durationParallelism }} ` + -FallbackTestDurationSeconds ${{ parameters.fallbackTestDurationSeconds }} ` + -NUnitWorkers '${{ parameters.nunitWorkers }}' ` + -Configuration '$(XA.Build.Configuration)' ` + -AndroidHome $androidHome ` + -JavaHome $env:JAVA_HOME ` + -NuGetPackages $env:NUGET_PACKAGES ` + -DotNetToolsDirectory '$(Agent.ToolsDirectory)' ` + -GradleUserHome (Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle') + displayName: Prepare duration-balanced ${{ parameters.platform }} Helix work items + +- template: /build-tools/automation/yaml-templates/run-dotnet-preview.yaml + parameters: + command: msbuild + project: $(System.DefaultWorkingDirectory)/build-tools/automation/HostBuildTestsHelix.proj + arguments: >- + /restore + /t:Test + /p:TreatWarningsAsErrors=false + /bl:$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results/send-to-helix.binlog + displayName: Submit ${{ parameters.platform }} host build tests to Helix + continueOnError: false + taskTimeoutInMinutes: 240 + env: + HostBuildTestsHelixWorkItemsProps: $(HostBuildTestsHelixWorkItemsProps) + CorrelationPayloadDirectory: $(HostBuildTestsHelixCorrelationPayload) + HostBuildTestsHelixWorkItemTimeout: ${{ parameters.workItemTimeout }} + HelixResultsDestinationDir: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results + HelixSource: pr/dotnet/android + HelixType: tests/android/host-build-tests/${{ parameters.platform }}/ + HelixBuild: $(Build.BuildNumber) + HelixConfiguration: ${{ parameters.platform }} + HelixTargetQueueForItem: ${{ parameters.helixQueue }} + HelixTestRunName: ${{ parameters.testRunTitle }} + HelixAccessToken: '' + WaitForWorkItemCompletion: 'true' + Creator: dotnet-android + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +- task: PublishTestResults@2 + displayName: Publish ${{ parameters.platform }} Helix TRX results + condition: always() + inputs: + testResultsFormat: VSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results + mergeTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + continueOnError: true + +- pwsh: | + $ErrorActionPreference = 'Stop' + $resultsDirectory = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results' + $timingDirectory = '$(Build.StagingDirectory)/host-build-test-timings-${{ parameters.platform }}' + $trxDirectory = Join-Path $timingDirectory 'trx' + New-Item -ItemType Directory -Force -Path $trxDirectory | Out-Null + + $trxFiles = @(Get-ChildItem -LiteralPath $resultsDirectory -Recurse -Filter '*.trx' -ErrorAction Ignore) + if ($trxFiles.Count -eq 0) { + Write-Warning "No TRX files were available to produce next-run timings." + Write-Host '##vso[task.setvariable variable=HostBuildTestsHelixHasTimings]false' + exit 0 + } + + for ($index = 0; $index -lt $trxFiles.Count; $index++) { + Copy-Item -LiteralPath $trxFiles[$index].FullName -Destination (Join-Path $trxDirectory ('{0:d4}.trx' -f $index)) + } + + $testSlicer = Join-Path '$(Agent.ToolsDirectory)' $(if ('${{ parameters.platform }}' -eq 'windows') { 'dotnet-test-slicer.exe' } else { 'dotnet-test-slicer' }) + & $testSlicer balance --input-directory=$trxDirectory --outfile=(Join-Path $timingDirectory 'balance.xml') + if ($LASTEXITCODE -ne 0) { + throw "dotnet-test-slicer balance failed with exit code $LASTEXITCODE." + } + Remove-Item -LiteralPath $trxDirectory -Recurse -Force + Write-Host '##vso[task.setvariable variable=HostBuildTestsHelixHasTimings]true' + displayName: Produce next-run ${{ parameters.platform }} timings + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.platform }} host-test timings + condition: and(always(), eq(variables['HostBuildTestsHelixHasTimings'], 'true')) + inputs: + artifactName: ${{ parameters.timingArtifactName }} + targetPath: $(Build.StagingDirectory)/host-build-test-timings-${{ parameters.platform }} + +- task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.platform }} Helix diagnostics + condition: always() + inputs: + artifactName: Host Build Test Helix Results - ${{ parameters.platform }} + targetPath: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results From 7b59cb7e7ad1859b41d111670d16571db0ff4c6d Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 15:45:51 -0700 Subject: [PATCH 2/8] AB#12334 Link Helix prototype tracking Document the Azure DevOps task that tracks the guarded host build test prototype. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/host-build-tests-helix/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index 905b40f0cc6..a4951251cd3 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -1,5 +1,7 @@ # Host build tests in Helix prototype +Tracking: [AB#12334](https://dev.azure.com/dnceng/internal/_workitems/edit/12334) + This guarded prototype stages the already-prepared .NET SDK/workload, Android SDK, JDK, NuGet/Gradle caches, selected repository inputs, and test assemblies into one Helix correlation payload per host OS. Work item payloads contain only a runsettings From c41b6aa050b1145daa26ddfa83d4a19dd38ba75f Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 17:20:56 -0700 Subject: [PATCH 3/8] AB#12334 Fix Linux Helix test discovery Use the NUnit class filter for BuildTest, fail clearly when discovery returns the slicer dummy test, preserve single-test arrays, and report staged payload size. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/azure-pipelines-public.yaml | 2 +- .../HostBuildTestsHelix.Tests.ps1 | 17 +++++++++++++++++ .../HostBuildTestsHelix.psm1 | 5 ++++- .../prepare-host-build-test-helix-payload.ps1 | 3 +++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 143ca87f891..946a5959a5c 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -232,7 +232,7 @@ stages: parameters: platform: linux helixQueue: Ubuntu.2404.Amd64.Open - testFilter: 'Name = BuildTest' + testFilter: 'class == Xamarin.Android.Build.Tests.BuildTest' testRunTitle: Xamarin.Android.Build.Tests - Linux BuildTest Helix Prototype timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} timingArtifactName: Host Build Test Timings - linux diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index 3d6bb23cf40..2d87470c77e 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -80,6 +80,23 @@ try { $roundTrippedNames = Get-NUnitRunSettingsTestNames -Path $runSettingsPath Assert-Equal ($specialNames -join '|') ($roundTrippedNames -join '|') 'Runsettings names should round trip.' + $singleRunSettingsPath = Join-Path $testRoot 'single.runsettings' + Write-NUnitRunSettings -TestNames @('OnlyTest') -Path $singleRunSettingsPath + $singleTest = Get-NUnitRunSettingsTestNames -Path $singleRunSettingsPath + Assert-Equal 1 $singleTest.Count 'A one-test runsettings file should remain an array.' + Assert-Equal 'OnlyTest' $singleTest[0] 'The single test should round trip.' + + $emptyRunSettingsPath = Join-Path $testRoot 'empty.runsettings' + Set-Content -LiteralPath $emptyRunSettingsPath -Value "test == 'dotnet-slicer-dummy-test-name'" + try { + Get-NUnitRunSettingsTestNames -Path $emptyRunSettingsPath + throw 'The slicer dummy test should have caused an error.' + } catch { + if ($_.Exception.Message -notlike '*matched no tests*') { + throw + } + } + $secondFallbackPlan = New-DurationBalancedWorkItems ` -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` -TimingHistory $emptyHistory ` diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index f51626c48b9..ecfe96052bf 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -51,8 +51,11 @@ function Get-NUnitRunSettingsTestNames if ($names.Count -eq 0) { throw "No tests were found in '$Path'." } + if ($names.Count -eq 1 -and $names[0] -eq 'dotnet-slicer-dummy-test-name') { + throw "The test filter matched no tests in '$Path'." + } - return $names.ToArray() + return ,$names.ToArray() } function Get-HostBuildTestTimingHistory diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 23c779f4764..6d9fefb795c 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -177,6 +177,9 @@ if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { -Optional } +$payloadBytes = (Get-ChildItem -LiteralPath $correlationPayloadFullPath -Recurse -File | Measure-Object -Property Length -Sum).Sum +Write-Host ('Staged correlation payload size: {0:N2} GiB.' -f ($payloadBytes / 1GB)) + $stagedTestAssembly = Join-Path $payloadRepository (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) if (-not (Test-Path -LiteralPath $stagedTestAssembly -PathType Leaf)) { throw "Test assembly '$stagedTestAssembly' was not staged." From 112d25143fbd141895dc6daeeeb7d25930aa5949 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 17:33:46 -0700 Subject: [PATCH 4/8] AB#12334 Preserve Helix work item metadata Let MSBuild copy generated Command and PayloadDirectory metadata when converting private work items to HelixWorkItem entries, while adding the shared timeout. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/HostBuildTestsHelix.proj | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj index db68078dd58..92dfcdeb5ce 100644 --- a/build-tools/automation/HostBuildTestsHelix.proj +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -25,8 +25,6 @@ - %(_HostBuildTestHelixWorkItem.PayloadDirectory) - %(_HostBuildTestHelixWorkItem.Command) $(HostBuildTestsHelixWorkItemTimeout) From 911c6b3cc5a8472b10d8f16158d78753985e8602 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 18:32:30 -0700 Subject: [PATCH 5/8] AB#12334 Partition Helix correlation payloads Split large toolchain trees into correlation payloads capped at 1 GiB, preserve their destinations through generated MSBuild items, and omit the duplicated NuGet cache to avoid the Helix SDK in-memory ZIP limit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/HostBuildTestsHelix.proj | 13 +- .../HostBuildTestsHelix.psm1 | 7 +- .../host-build-tests-helix/README.md | 12 +- .../prepare-host-build-test-helix-payload.ps1 | 230 +++++++++++++++--- .../run-host-build-tests-in-helix.yaml | 5 +- 5 files changed, 220 insertions(+), 47 deletions(-) diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj index 92dfcdeb5ce..1545f015658 100644 --- a/build-tools/automation/HostBuildTestsHelix.proj +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -10,6 +10,7 @@ + @@ -18,9 +19,7 @@ - - %(Identity) - + @@ -32,13 +31,15 @@ - + + - diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index ecfe96052bf..e1dafe4f35c 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -293,7 +293,7 @@ set "DOTNET_ROOT=%REPO%\bin\__CONFIGURATION__\dotnet" set "ANDROID_HOME=%HELIX_CORRELATION_PAYLOAD%\android-toolchain\sdk" set "ANDROID_SDK_ROOT=%ANDROID_HOME%" set "JAVA_HOME=%HELIX_CORRELATION_PAYLOAD%\jdk" -set "NUGET_PACKAGES=%HELIX_CORRELATION_PAYLOAD%\nuget-packages" +set "NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\nuget-packages" set "GRADLE_USER_HOME=%HELIX_CORRELATION_PAYLOAD%\gradle" set "DOTNET_CLI_HOME=%HELIX_WORKITEM_ROOT%\dotnet-home" set "TEMP=%HELIX_WORKITEM_ROOT%\temp" @@ -316,6 +316,7 @@ if not exist "%DOTNET_ROOT%\dotnet.exe" exit /b 3 if not exist "%REPO%\__TEST_ASSEMBLY__" exit /b 3 mkdir "%HELIX_WORKITEM_UPLOAD_ROOT%\dumps" 2>nul mkdir "%DOTNET_CLI_HOME%" 2>nul +mkdir "%NUGET_PACKAGES%" 2>nul mkdir "%TEMP%" 2>nul copy /y "%~dp0slice.runsettings" "%HELIX_WORKITEM_UPLOAD_ROOT%\slice.runsettings" >nul copy /y "%~dp0work-item.json" "%HELIX_WORKITEM_UPLOAD_ROOT%\work-item.json" >nul @@ -344,7 +345,7 @@ export DOTNET_ROOT="$REPO/bin/__CONFIGURATION__/dotnet" export ANDROID_HOME="$HELIX_CORRELATION_PAYLOAD/android-toolchain/sdk" export ANDROID_SDK_ROOT="$ANDROID_HOME" export JAVA_HOME="$HELIX_CORRELATION_PAYLOAD/jdk" -export NUGET_PACKAGES="$HELIX_CORRELATION_PAYLOAD/nuget-packages" +export NUGET_PACKAGES="$HELIX_WORKITEM_ROOT/nuget-packages" export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" export DOTNET_CLI_HOME="$HELIX_WORKITEM_ROOT/dotnet-home" export TMPDIR="$HELIX_WORKITEM_ROOT/temp" @@ -364,7 +365,7 @@ export DOTNET_DbgMiniDumpType=2 export DOTNET_DbgMiniDumpName="$HELIX_WORKITEM_UPLOAD_ROOT/dumps/%e.%p.dmp" test -x "$DOTNET_ROOT/dotnet" test -f "$REPO/__TEST_ASSEMBLY__" -mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$TMPDIR" +mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$NUGET_PACKAGES" "$TMPDIR" cp "$HELIX_WORKITEM_ROOT/slice.runsettings" "$HELIX_WORKITEM_UPLOAD_ROOT/slice.runsettings" cp "$HELIX_WORKITEM_ROOT/work-item.json" "$HELIX_WORKITEM_UPLOAD_ROOT/work-item.json" cd "$REPO" diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index a4951251cd3..4f35a3076b2 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -3,11 +3,13 @@ Tracking: [AB#12334](https://dev.azure.com/dnceng/internal/_workitems/edit/12334) This guarded prototype stages the already-prepared .NET SDK/workload, Android SDK, -JDK, NuGet/Gradle caches, selected repository inputs, and test assemblies into one -Helix correlation payload per host OS. Work item payloads contain only a runsettings -filter, a small command script, and generation metadata. To control payload size, the -source copy omits `src/Mono.Android`; host build tests consume that product through the -prepared workload packs instead. +JDK, selected Gradle caches, repository inputs, and test assemblies as shared Helix +correlation payloads. Individual payloads are capped at 1 GiB to avoid the Helix SDK's +in-memory ZIP limit. Work item payloads contain only a runsettings filter, a small +command script, and generation metadata. To control total payload size, the source +copy omits `src/Mono.Android` and the NuGet package cache; host build tests consume the +product through the prepared workload packs and use isolated per-work-item NuGet +caches. `prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's `dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 6d9fefb795c..9dbcece4b34 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -10,6 +10,9 @@ param ( [Parameter(Mandatory)] [string] $CorrelationPayloadDirectory, + [Parameter(Mandatory)] + [string] $CorrelationPayloadsPropsPath, + [Parameter(Mandatory)] [string] $WorkItemsDirectory, @@ -48,8 +51,6 @@ param ( [Parameter(Mandatory)] [string] $JavaHome, - [string] $NuGetPackages = '', - [string] $DotNetToolsDirectory = '', [string] $GradleUserHome = '' @@ -105,8 +106,167 @@ function Copy-PayloadDirectory } } +function Get-DirectorySize +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $size = 0L + foreach ($file in Get-ChildItem -LiteralPath $Path -Recurse -File) { + $size += $file.Length + } + return $size +} + +$correlationPayloads = [System.Collections.Generic.List[object]]::new() +$filePayloadIndex = 0 +$maximumPayloadBytes = 1GB + +function Add-CorrelationPayload +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [Parameter(Mandatory)] + [long] $SizeBytes + ) + + $script:correlationPayloads.Add([pscustomobject] @{ + Source = [IO.Path]::GetFullPath($Source) + Destination = $Destination.Replace('\', '/').Trim('/') + SizeBytes = $SizeBytes + }) +} + +function Add-FileCorrelationPayloads +{ + param ( + [Parameter(Mandatory)] + [IO.FileInfo[]] $Files, + + [Parameter(Mandatory)] + [string] $Destination + ) + + $currentFiles = [System.Collections.Generic.List[IO.FileInfo]]::new() + $currentSize = 0L + foreach ($file in $Files | Sort-Object Name) { + if ($file.Length -gt $script:maximumPayloadBytes) { + throw "File '$($file.FullName)' exceeds the maximum correlation payload size." + } + if ($currentFiles.Count -gt 0 -and $currentSize + $file.Length -gt $script:maximumPayloadBytes) { + New-FileCorrelationPayload -Files $currentFiles.ToArray() -Destination $Destination -SizeBytes $currentSize + $currentFiles.Clear() + $currentSize = 0L + } + $currentFiles.Add($file) + $currentSize += $file.Length + } + if ($currentFiles.Count -gt 0) { + New-FileCorrelationPayload -Files $currentFiles.ToArray() -Destination $Destination -SizeBytes $currentSize + } +} + +function New-FileCorrelationPayload +{ + param ( + [Parameter(Mandatory)] + [IO.FileInfo[]] $Files, + + [Parameter(Mandatory)] + [string] $Destination, + + [Parameter(Mandatory)] + [long] $SizeBytes + ) + + $script:filePayloadIndex++ + $payloadDirectory = Join-Path $script:correlationPayloadFullPath ('files-{0:d3}' -f $script:filePayloadIndex) + New-Item -ItemType Directory -Force -Path $payloadDirectory | Out-Null + foreach ($file in $Files) { + $target = Join-Path $payloadDirectory $file.Name + try { + New-Item -ItemType HardLink -Path $target -Target $file.FullName | Out-Null + } catch { + Copy-Item -LiteralPath $file.FullName -Destination $target + } + } + Add-CorrelationPayload -Source $payloadDirectory -Destination $Destination -SizeBytes $SizeBytes +} + +function Add-DirectoryCorrelationPayloads +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [switch] $Optional + ) + + if (-not (Test-Path -LiteralPath $Source -PathType Container)) { + if ($Optional) { + Write-Warning "Optional correlation payload directory '$Source' does not exist." + return + } + throw "Required correlation payload directory '$Source' does not exist." + } + + $size = Get-DirectorySize -Path $Source + if ($size -le $script:maximumPayloadBytes) { + Add-CorrelationPayload -Source $Source -Destination $Destination -SizeBytes $size + return + } + + $files = @(Get-ChildItem -LiteralPath $Source -File) + if ($files.Count -gt 0) { + Add-FileCorrelationPayloads -Files $files -Destination $Destination + } + foreach ($directory in Get-ChildItem -LiteralPath $Source -Directory | Sort-Object Name) { + Add-DirectoryCorrelationPayloads ` + -Source $directory.FullName ` + -Destination "$($Destination.TrimEnd('/'))/$($directory.Name)" + } +} + +function Write-CorrelationPayloadProps +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $settings = [Xml.XmlWriterSettings]::new() + $settings.Indent = $true + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $writer = [Xml.XmlWriter]::Create($Path, $settings) + try { + $writer.WriteStartElement('Project') + $writer.WriteStartElement('ItemGroup') + foreach ($payload in $script:correlationPayloads) { + $writer.WriteStartElement('_HostBuildTestHelixCorrelationPayload') + $writer.WriteAttributeString('Include', $payload.Source) + $writer.WriteElementString('Destination', $payload.Destination) + $writer.WriteEndElement() + } + $writer.WriteEndElement() + $writer.WriteEndElement() + } finally { + $writer.Dispose() + } +} + $repositoryRootFullPath = [IO.Path]::GetFullPath($RepositoryRoot) $correlationPayloadFullPath = [IO.Path]::GetFullPath($CorrelationPayloadDirectory) +$correlationPayloadsPropsFullPath = [IO.Path]::GetFullPath($CorrelationPayloadsPropsPath) $workItemsFullPath = [IO.Path]::GetFullPath($WorkItemsDirectory) $workItemsPropsFullPath = [IO.Path]::GetFullPath($WorkItemsPropsPath) $resultsFullPath = [IO.Path]::GetFullPath($ResultsDirectory) @@ -115,13 +275,11 @@ Remove-Item -LiteralPath $correlationPayloadFullPath -Recurse -Force -ErrorActio Remove-Item -LiteralPath $workItemsFullPath -Recurse -Force -ErrorAction Ignore New-Item -ItemType Directory -Force -Path $correlationPayloadFullPath, $workItemsFullPath, $resultsFullPath | Out-Null -$payloadRepository = Join-Path $correlationPayloadFullPath 'repo' +$payloadRepository = Join-Path $correlationPayloadFullPath 'repository' New-Item -ItemType Directory -Force -Path $payloadRepository | Out-Null Get-ChildItem -LiteralPath $repositoryRootFullPath -File | Copy-Item -Destination $payloadRepository $payloadDirectories = @( - @("bin\$Configuration", "bin\$Configuration", $false), - @("bin\Test$Configuration", "bin\Test$Configuration", $false), @('build-tools', 'build-tools', $false), @('eng', 'eng', $false), @('external', 'external', $false), @@ -145,26 +303,8 @@ foreach ($sourceDirectory in Get-ChildItem -LiteralPath $sourceRoot -Directory | -Destination (Join-Path (Join-Path $payloadRepository 'src') $sourceDirectory.Name) } -Copy-PayloadDirectory -Source $AndroidHome -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath 'android-toolchain/sdk')) -Copy-PayloadDirectory -Source $JavaHome -Destination (Join-Path $correlationPayloadFullPath 'jdk') - -if ([string]::IsNullOrWhiteSpace($NuGetPackages)) { - $NuGetPackages = Join-Path ([Environment]::GetFolderPath('UserProfile')) (ConvertTo-NativeRelativePath '.nuget/packages') -} -Copy-PayloadDirectory -Source $NuGetPackages -Destination (Join-Path $correlationPayloadFullPath 'nuget-packages') - -if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { - $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' -} -foreach ($gradleDirectory in @('caches', 'wrapper')) { - Copy-PayloadDirectory ` - -Source (Join-Path $GradleUserHome $gradleDirectory) ` - -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath "gradle/$gradleDirectory")) ` - -Optional -} - +$payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { - $payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' New-Item -ItemType Directory -Force -Path $payloadTools | Out-Null $apkDiffShim = if ($Platform -eq 'windows') { 'apkdiff.exe' } else { 'apkdiff' } $apkDiffShimPath = Join-Path $DotNetToolsDirectory $apkDiffShim @@ -177,12 +317,40 @@ if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { -Optional } -$payloadBytes = (Get-ChildItem -LiteralPath $correlationPayloadFullPath -Recurse -File | Measure-Object -Property Length -Sum).Sum -Write-Host ('Staged correlation payload size: {0:N2} GiB.' -f ($payloadBytes / 1GB)) +Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` + -Destination "repo/bin/$Configuration" +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration")) ` + -Destination "repo/bin/Test$Configuration" +Add-DirectoryCorrelationPayloads -Source $AndroidHome -Destination 'android-toolchain/sdk' +Add-DirectoryCorrelationPayloads -Source $JavaHome -Destination 'jdk' + +if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { + $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' +} +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'wrapper/dists')) ` + -Destination 'gradle/wrapper/dists' ` + -Optional +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'caches/modules-2')) ` + -Destination 'gradle/caches/modules-2' ` + -Optional +if (Test-Path -LiteralPath $payloadTools -PathType Container) { + Add-DirectoryCorrelationPayloads -Source $payloadTools -Destination 'dotnet-tools' +} + +Write-CorrelationPayloadProps -Path $correlationPayloadsPropsFullPath +$payloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Sum).Sum +$largestPayloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Maximum).Maximum +Write-Host ('Prepared {0} correlation payloads totaling {1:N2} GiB; largest payload is {2:N2} GiB.' -f ` + $correlationPayloads.Count, ($payloadBytes / 1GB), ($largestPayloadBytes / 1GB)) -$stagedTestAssembly = Join-Path $payloadRepository (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) -if (-not (Test-Path -LiteralPath $stagedTestAssembly -PathType Leaf)) { - throw "Test assembly '$stagedTestAssembly' was not staged." +$testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) +if (-not (Test-Path -LiteralPath $testAssembly -PathType Leaf)) { + throw "Test assembly '$testAssembly' does not exist." } if (-not (Test-Path -LiteralPath $TestSlicerPath -PathType Leaf)) { throw "dotnet-test-slicer '$TestSlicerPath' does not exist." @@ -191,7 +359,7 @@ if (-not (Test-Path -LiteralPath $TestSlicerPath -PathType Leaf)) { $discoveryRunSettings = Join-Path $workItemsFullPath 'all-tests.runsettings' $sliceArguments = @( 'slice', - "--test-assembly=$stagedTestAssembly", + "--test-assembly=$testAssembly", '--slice-number=1', '--total-slices=1', "--outfile=$discoveryRunSettings" @@ -254,6 +422,6 @@ if ($plan.DurationMode -eq 'count-fallback') { Write-Warning "No matching timing history was available. Work items use an explicit count fallback of $FallbackTestDurationSeconds second(s) per test." } -Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixCorrelationPayload]$correlationPayloadFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixCorrelationPayloadsProps]$correlationPayloadsPropsFullPath" Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemsProps]$workItemsPropsFullPath" Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemCount]$($plan.WorkItems.Count)" diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml index 6132358d98e..e6c95925e8a 100644 --- a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -57,6 +57,7 @@ steps: $stagingRoot = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}' $resultsDirectory = Join-Path $stagingRoot 'results' $correlationPayload = Join-Path $stagingRoot 'correlation' + $correlationPayloadsProps = Join-Path $stagingRoot 'correlation-payloads.props' $workItemsDirectory = Join-Path $stagingRoot 'work-items' $workItemsProps = Join-Path $stagingRoot 'work-items.props' $adbName = if ($platform -eq 'windows') { 'adb.exe' } else { 'adb' } @@ -75,6 +76,7 @@ steps: -Platform $platform ` -RepositoryRoot '$(System.DefaultWorkingDirectory)' ` -CorrelationPayloadDirectory $correlationPayload ` + -CorrelationPayloadsPropsPath $correlationPayloadsProps ` -WorkItemsDirectory $workItemsDirectory ` -WorkItemsPropsPath $workItemsProps ` -ResultsDirectory $resultsDirectory ` @@ -89,7 +91,6 @@ steps: -Configuration '$(XA.Build.Configuration)' ` -AndroidHome $androidHome ` -JavaHome $env:JAVA_HOME ` - -NuGetPackages $env:NUGET_PACKAGES ` -DotNetToolsDirectory '$(Agent.ToolsDirectory)' ` -GradleUserHome (Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle') displayName: Prepare duration-balanced ${{ parameters.platform }} Helix work items @@ -108,7 +109,7 @@ steps: taskTimeoutInMinutes: 240 env: HostBuildTestsHelixWorkItemsProps: $(HostBuildTestsHelixWorkItemsProps) - CorrelationPayloadDirectory: $(HostBuildTestsHelixCorrelationPayload) + HostBuildTestsHelixCorrelationPayloadsProps: $(HostBuildTestsHelixCorrelationPayloadsProps) HostBuildTestsHelixWorkItemTimeout: ${{ parameters.workItemTimeout }} HelixResultsDestinationDir: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results HelixSource: pr/dotnet/android From 39d7209de44d4caad6eb4d2e4ff344c16322ac42 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 19:30:24 -0700 Subject: [PATCH 6/8] AB#12334 Exclude device-only Android payloads Keep emulator images, skins, and sources out of host-only Helix correlation payloads so large system images are neither uploaded nor downloaded by build-test work items. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../prepare-host-build-test-helix-payload.ps1 | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 9dbcece4b34..776e7049929 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -324,7 +324,20 @@ Add-DirectoryCorrelationPayloads ` Add-DirectoryCorrelationPayloads ` -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration")) ` -Destination "repo/bin/Test$Configuration" -Add-DirectoryCorrelationPayloads -Source $AndroidHome -Destination 'android-toolchain/sdk' + +$androidRootFiles = @(Get-ChildItem -LiteralPath $AndroidHome -File) +if ($androidRootFiles.Count -gt 0) { + Add-FileCorrelationPayloads -Files $androidRootFiles -Destination 'android-toolchain/sdk' +} +$excludedAndroidDirectories = @('emulator', 'skins', 'sources', 'system-images') +foreach ($androidDirectory in Get-ChildItem -LiteralPath $AndroidHome -Directory | Sort-Object Name) { + if ($androidDirectory.Name -notin $excludedAndroidDirectories) { + Add-DirectoryCorrelationPayloads ` + -Source $androidDirectory.FullName ` + -Destination "android-toolchain/sdk/$($androidDirectory.Name)" + } +} + Add-DirectoryCorrelationPayloads -Source $JavaHome -Destination 'jdk' if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { From 7802c120276fe1bc139436e256b806ac99cf4696 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 21:09:16 -0700 Subject: [PATCH 7/8] AB#12334 Report Helix payload composition Log and persist the largest correlation payload destinations so setup size can be reduced based on measured CI data rather than aggregate bytes alone. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../prepare-host-build-test-helix-payload.ps1 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 776e7049929..bf10c33c0a9 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -360,6 +360,13 @@ $payloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Sum) $largestPayloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Maximum).Maximum Write-Host ('Prepared {0} correlation payloads totaling {1:N2} GiB; largest payload is {2:N2} GiB.' -f ` $correlationPayloads.Count, ($payloadBytes / 1GB), ($largestPayloadBytes / 1GB)) +Write-Host 'Largest correlation payload components:' +$correlationPayloads | + Sort-Object SizeBytes -Descending | + Select-Object -First 20 | + ForEach-Object { + Write-Host ('- {0:N2} GiB -> {1}' -f ($_.SizeBytes / 1GB), $_.Destination) + } $testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) if (-not (Test-Path -LiteralPath $testAssembly -PathType Leaf)) { @@ -417,6 +424,14 @@ $summaryPath = Join-Path $resultsFullPath 'work-item-generation.json' estimatedTestCount = $plan.EstimatedTestCount workItemCount = $plan.WorkItems.Count totalDurationLoadMs = $plan.TotalDurationMs + correlationPayloadCount = $correlationPayloads.Count + correlationPayloadBytes = $payloadBytes + correlationPayloads = @($correlationPayloads | ForEach-Object { + [pscustomobject] @{ + destination = $_.Destination + sizeBytes = $_.SizeBytes + } + }) workItems = @($plan.WorkItems | ForEach-Object { [pscustomobject] @{ id = $_.Id From c6114de22c3e35515d4b55fdbf1c9234e998045f Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 21:58:01 -0700 Subject: [PATCH 8/8] AB#12334 Trim host-test payload inputs Exclude SDK docs/extras and the Java.Interop checkout, stage only the net10 host test assembly inputs, and add an opt-in host-only iteration mode while preserving full dual execution for parity runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/azure-pipelines-public.yaml | 6 ++++- .../host-build-tests-helix/README.md | 3 ++- .../prepare-host-build-test-helix-payload.ps1 | 23 ++++++++++++++----- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 946a5959a5c..c162f29acc9 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -55,6 +55,10 @@ parameters: displayName: Host build tests Helix work item timeout type: string default: '00:45:00' +- name: hostBuildTestsHelixOnly + displayName: Run only the host build tests Helix prototype after builds + type: boolean + default: false # Repository resources resources: @@ -193,7 +197,7 @@ stages: dependsOn: - mac_build - linux_build - condition: and(succeeded(), ne(variables['SkipTestStages'], 'true')) + condition: and(succeeded(), or(ne(variables['SkipTestStages'], 'true'), eq('${{ parameters.hostBuildTestsHelixOnly }}', 'true'))) jobs: - job: windows_host_build_tests_helix displayName: Windows > Host Build Tests > Helix diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index 4f35a3076b2..476dcc348d4 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -9,7 +9,8 @@ in-memory ZIP limit. Work item payloads contain only a runsettings filter, a sma command script, and generation metadata. To control total payload size, the source copy omits `src/Mono.Android` and the NuGet package cache; host build tests consume the product through the prepared workload packs and use isolated per-work-item NuGet -caches. +caches. Device-only Android SDK content, SDK documentation, the Java.Interop checkout, +and unrelated test assemblies are also excluded. `prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's `dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index bf10c33c0a9..7dc5d590fdb 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -282,7 +282,6 @@ Get-ChildItem -LiteralPath $repositoryRootFullPath -File | Copy-Item -Destinatio $payloadDirectories = @( @('build-tools', 'build-tools', $false), @('eng', 'eng', $false), - @('external', 'external', $false), @('.github', '.github', $true), @('samples', 'samples', $true), @('src-ThirdParty', 'src-ThirdParty', $true), @@ -321,15 +320,28 @@ Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' Add-DirectoryCorrelationPayloads ` -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` -Destination "repo/bin/$Configuration" -Add-DirectoryCorrelationPayloads ` - -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration")) ` - -Destination "repo/bin/Test$Configuration" + +$testOutputRoot = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration") +$testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) +$testAssemblyDirectory = Split-Path -Parent $testAssembly +$testAssemblyDestination = "repo/bin/Test$Configuration/$([IO.Path]::GetRelativePath($testOutputRoot, $testAssemblyDirectory).Replace('\', '/'))" +Add-DirectoryCorrelationPayloads -Source $testAssemblyDirectory -Destination $testAssemblyDestination +$testOutputFiles = @(Get-ChildItem -LiteralPath $testOutputRoot -File) +if ($testOutputFiles.Count -gt 0) { + Add-FileCorrelationPayloads -Files $testOutputFiles -Destination "repo/bin/Test$Configuration" +} +foreach ($testDirectory in Get-ChildItem -LiteralPath $testOutputRoot -Directory | + Where-Object { $_.Name -eq 'Expected' -or $_.Name -like 'android-*' }) { + Add-DirectoryCorrelationPayloads ` + -Source $testDirectory.FullName ` + -Destination "repo/bin/Test$Configuration/$($testDirectory.Name)" +} $androidRootFiles = @(Get-ChildItem -LiteralPath $AndroidHome -File) if ($androidRootFiles.Count -gt 0) { Add-FileCorrelationPayloads -Files $androidRootFiles -Destination 'android-toolchain/sdk' } -$excludedAndroidDirectories = @('emulator', 'skins', 'sources', 'system-images') +$excludedAndroidDirectories = @('docs', 'emulator', 'extras', 'skins', 'sources', 'system-images') foreach ($androidDirectory in Get-ChildItem -LiteralPath $AndroidHome -Directory | Sort-Object Name) { if ($androidDirectory.Name -notin $excludedAndroidDirectories) { Add-DirectoryCorrelationPayloads ` @@ -368,7 +380,6 @@ $correlationPayloads | Write-Host ('- {0:N2} GiB -> {1}' -f ($_.SizeBytes / 1GB), $_.Destination) } -$testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) if (-not (Test-Path -LiteralPath $testAssembly -PathType Leaf)) { throw "Test assembly '$testAssembly' does not exist." }