diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj new file mode 100644 index 00000000000..1545f015658 --- /dev/null +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -0,0 +1,45 @@ + + + msbuild + true + false + true + tests/android/host-build-tests/ + Xamarin.Android.Build.Tests - Helix + 00:45:00 + + + + + + + + $(HelixTestRunName) + + + + + + + + + + $(HostBuildTestsHelixWorkItemTimeout) + + + + + + + + + + + + diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 44eae1ceb96..c162f29acc9 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -31,6 +31,34 @@ parameters: values: - 'true' - 'false' +- name: enableHostBuildTestsHelixPrototype + displayName: Enable host build tests in Helix prototype + type: boolean + default: false +- name: hostBuildTestsHelixTargetMinutes + displayName: Host build tests Helix target minutes + type: number + default: 15 +- name: hostBuildTestsHelixDurationParallelism + displayName: Host build tests timing parallelism + type: number + default: 2.5 +- name: hostBuildTestsHelixFallbackTestDurationSeconds + displayName: Host build tests fallback seconds per test + type: number + default: 60 +- name: hostBuildTestsHelixTimingBuildId + displayName: Prior build ID containing host build test timings + type: string + default: '' +- name: hostBuildTestsHelixWorkItemTimeout + displayName: Host build tests Helix work item timeout + type: string + default: '00:45:00' +- name: hostBuildTestsHelixOnly + displayName: Run only the host build tests Helix prototype after builds + type: boolean + default: false # Repository resources resources: @@ -162,6 +190,61 @@ stages: nugetArtifactName: $(LinuxNuGetArtifactName) use1ESTemplate: false +# Guarded additive prototype. The existing Windows and Linux test jobs remain enabled. +- ${{ if eq(parameters.enableHostBuildTestsHelixPrototype, true) }}: + - stage: host_build_tests_helix + displayName: Host Build Tests Helix Prototype + dependsOn: + - mac_build + - linux_build + condition: and(succeeded(), or(ne(variables['SkipTestStages'], 'true'), eq('${{ parameters.hostBuildTestsHelixOnly }}', 'true'))) + jobs: + - job: windows_host_build_tests_helix + displayName: Windows > Host Build Tests > Helix + pool: + name: $(NetCorePublicPoolName) + demands: + - ImageOverride -equals $(WindowsPoolImageNetCorePublic) + timeoutInMinutes: 300 + workspace: + clean: all + steps: + - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml + parameters: + platform: windows + helixQueue: Windows.Amd64.VS2026.Open + testFilter: '' + testRunTitle: Xamarin.Android.Build.Tests - Windows Helix Prototype + timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} + timingArtifactName: Host Build Test Timings - windows + targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }} + durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }} + fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }} + workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }} + + - job: linux_host_build_tests_helix + displayName: Linux > BuildTest > Helix + pool: + name: $(NetCorePublicPoolName) + demands: + - ImageOverride -equals $(LinuxPoolImageNetCorePublic) + timeoutInMinutes: 300 + workspace: + clean: all + steps: + - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml + parameters: + platform: linux + helixQueue: Ubuntu.2404.Amd64.Open + testFilter: 'class == Xamarin.Android.Build.Tests.BuildTest' + testRunTitle: Xamarin.Android.Build.Tests - Linux BuildTest Helix Prototype + timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} + timingArtifactName: Host Build Test Timings - linux + targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }} + durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }} + fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }} + workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }} + # Package Tests Stage - ${{ if ne(variables.SkipTestStages, 'true') }}: - template: /build-tools/automation/yaml-templates/stage-package-tests.yaml diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 new file mode 100644 index 00000000000..2d87470c77e --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -0,0 +1,152 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'HostBuildTestsHelix.psm1') -Force + +function Assert-Equal +{ + param ( + [Parameter(Mandatory)] + [object] $Expected, + + [Parameter(Mandatory)] + [object] $Actual, + + [Parameter(Mandatory)] + [string] $Message + ) + + if ($Expected -ne $Actual) { + throw "$Message Expected '$Expected', got '$Actual'." + } +} + +$testRoot = Join-Path $PSScriptRoot '.test-output' +Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore +New-Item -ItemType Directory -Force -Path $testRoot | Out-Null + +try { + $emptyHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $fallbackPlan = New-DurationBalancedWorkItems ` + -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` + -TimingHistory $emptyHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 ` + -FallbackTestDurationSeconds 30 + + Assert-Equal 'count-fallback' $fallbackPlan.DurationMode 'No-history mode should be explicit.' + Assert-Equal 3 $fallbackPlan.WorkItems.Count 'Equal fallback durations should create three full bins.' + foreach ($workItem in $fallbackPlan.WorkItems) { + Assert-Equal 2 $workItem.Tests.Count 'Each fallback bin should contain two tests.' + Assert-Equal 60000 $workItem.EstimatedDurationMs 'Each fallback bin should target one minute.' + } + + $history = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $history.Add('Oversized', 720000) + $history.Add('Eight', 480000) + $history.Add('Six', 360000) + $history.Add('Four', 240000) + $historicalPlan = New-DurationBalancedWorkItems ` + -TestNames @('Four', 'Six', 'Eight', 'Oversized') ` + -TimingHistory $history ` + -TargetMinutes 10 ` + -DurationParallelism 1 + + Assert-Equal 'historical' $historicalPlan.DurationMode 'Complete history should be reported.' + Assert-Equal 3 $historicalPlan.WorkItems.Count 'Best-fit packing should create three bins.' + $oversized = @($historicalPlan.WorkItems | Where-Object Oversized) + Assert-Equal 1 $oversized.Count 'Only the over-target test should be oversized.' + Assert-Equal 1 $oversized[0].Tests.Count 'An over-target test must remain isolated.' + + $partialHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $partialHistory.Add('KnownOne', 10000) + $partialHistory.Add('KnownTwo', 30000) + $partialPlan = New-DurationBalancedWorkItems ` + -TestNames @('KnownOne', 'KnownTwo', 'NewTest') ` + -TimingHistory $partialHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 + + Assert-Equal 'partial-history' $partialPlan.DurationMode 'Missing tests should be reported as partial history.' + Assert-Equal 20000 $partialPlan.MissingTestDurationMs 'Missing tests should use the selected-history average.' + + $zeroTimingPath = Join-Path $testRoot 'zero-timing.xml' + Set-Content -LiteralPath $zeroTimingPath -Value '' + $zeroTimingHistory = Get-HostBuildTestTimingHistory -Path $zeroTimingPath + Assert-Equal 1 $zeroTimingHistory['FastTest'] 'Sub-millisecond TRX timings should retain a minimal positive load.' + + $specialNames = @("Namespace.Test('value')", 'Namespace.Path(C:\repo\file)') + $runSettingsPath = Join-Path $testRoot 'special.runsettings' + Write-NUnitRunSettings -TestNames $specialNames -Path $runSettingsPath + $roundTrippedNames = Get-NUnitRunSettingsTestNames -Path $runSettingsPath + Assert-Equal ($specialNames -join '|') ($roundTrippedNames -join '|') 'Runsettings names should round trip.' + + $singleRunSettingsPath = Join-Path $testRoot 'single.runsettings' + Write-NUnitRunSettings -TestNames @('OnlyTest') -Path $singleRunSettingsPath + $singleTest = Get-NUnitRunSettingsTestNames -Path $singleRunSettingsPath + Assert-Equal 1 $singleTest.Count 'A one-test runsettings file should remain an array.' + Assert-Equal 'OnlyTest' $singleTest[0] 'The single test should round trip.' + + $emptyRunSettingsPath = Join-Path $testRoot 'empty.runsettings' + Set-Content -LiteralPath $emptyRunSettingsPath -Value "test == 'dotnet-slicer-dummy-test-name'" + try { + Get-NUnitRunSettingsTestNames -Path $emptyRunSettingsPath + throw 'The slicer dummy test should have caused an error.' + } catch { + if ($_.Exception.Message -notlike '*matched no tests*') { + throw + } + } + + $secondFallbackPlan = New-DurationBalancedWorkItems ` + -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` + -TimingHistory $emptyHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 ` + -FallbackTestDurationSeconds 30 + Assert-Equal ` + ($fallbackPlan | ConvertTo-Json -Depth 8 -Compress) ` + ($secondFallbackPlan | ConvertTo-Json -Depth 8 -Compress) ` + 'Work-item generation should be deterministic.' + + $payloadDirectory = Join-Path $testRoot 'payloads' + $propsPath = Join-Path $testRoot 'work-items.props' + Write-HostBuildTestWorkItemPayloads ` + -Platform windows ` + -Plan $fallbackPlan ` + -WorkItemsDirectory $payloadDirectory ` + -WorkItemsPropsPath $propsPath ` + -TestAssemblyRelativePath 'bin\TestRelease\net10.0\Xamarin.Android.Build.Tests.dll' ` + -NUnitWorkers '-1' ` + -Configuration Release + + [xml] $props = Get-Content -LiteralPath $propsPath -Raw + Assert-Equal 3 @($props.Project.ItemGroup._HostBuildTestHelixWorkItem).Count 'Props should contain every generated work item.' + foreach ($item in @($props.Project.ItemGroup._HostBuildTestHelixWorkItem)) { + if (-not (Test-Path -LiteralPath $item.PayloadDirectory -PathType Container)) { + throw "Payload directory '$($item.PayloadDirectory)' was not created." + } + $command = Get-Content -LiteralPath (Join-Path $item.PayloadDirectory 'run-host-tests.cmd') -Raw + if ($command.Contains('__CONFIGURATION__') -or -not $command.Contains('bin\Release\dotnet')) { + throw 'The generated command did not apply the requested build configuration.' + } + } + + $linuxPayloadDirectory = Join-Path $testRoot 'linux-payloads' + $linuxPropsPath = Join-Path $testRoot 'linux-work-items.props' + Write-HostBuildTestWorkItemPayloads ` + -Platform linux ` + -Plan $fallbackPlan ` + -WorkItemsDirectory $linuxPayloadDirectory ` + -WorkItemsPropsPath $linuxPropsPath ` + -TestAssemblyRelativePath 'bin/TestRelease/net10.0/Xamarin.Android.Build.Tests.dll' ` + -NUnitWorkers '-1' ` + -Configuration Release + [xml] $linuxProps = Get-Content -LiteralPath $linuxPropsPath -Raw + foreach ($item in @($linuxProps.Project.ItemGroup._HostBuildTestHelixWorkItem)) { + Assert-Equal 'bash run-host-tests.sh' ([string] $item.Command) 'Linux work items should explicitly use bash.' + } + + Write-Host 'HostBuildTestsHelix tests passed.' +} finally { + Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore +} diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 new file mode 100644 index 00000000000..e1dafe4f35c --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -0,0 +1,472 @@ +Set-StrictMode -Version Latest + +function Get-NUnitRunSettingsTestNames +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + [xml] $runSettings = Get-Content -LiteralPath $Path -Raw + $where = [string] $runSettings.RunSettings.NUnit.Where + $prefix = "test == '" + $names = [System.Collections.Generic.List[string]]::new() + $offset = 0 + + while ($offset -lt $where.Length) { + $start = $where.IndexOf($prefix, $offset, [StringComparison]::Ordinal) + if ($start -lt 0) { + break + } + + $index = $start + $prefix.Length + $name = [Text.StringBuilder]::new() + $closed = $false + while ($index -lt $where.Length) { + $character = $where[$index] + if ($character -eq '\') { + if ($index + 1 -ge $where.Length) { + throw "Invalid escaped test name in '$Path'." + } + $index++ + [void] $name.Append($where[$index]) + } elseif ($character -eq "'") { + $closed = $true + $index++ + break + } else { + [void] $name.Append($character) + } + $index++ + } + + if (-not $closed) { + throw "Unterminated test name in '$Path'." + } + + $names.Add($name.ToString()) + $offset = $index + } + + if ($names.Count -eq 0) { + throw "No tests were found in '$Path'." + } + if ($names.Count -eq 1 -and $names[0] -eq 'dotnet-slicer-dummy-test-name') { + throw "The test filter matched no tests in '$Path'." + } + + return ,$names.ToArray() +} + +function Get-HostBuildTestTimingHistory +{ + param ( + [string] $Path + ) + + $durations = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + if ([string]::IsNullOrWhiteSpace($Path)) { + return $durations + } + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Timing history '$Path' does not exist." + } + + [xml] $balance = Get-Content -LiteralPath $Path -Raw + foreach ($test in @($balance.tests.test)) { + $name = [string] $test.name + $duration = 0L + if ([string]::IsNullOrWhiteSpace($name) -or + -not [long]::TryParse([string] $test.duration, [ref] $duration) -or + $duration -lt 0) { + throw "Timing history '$Path' contains an invalid test entry." + } + if (-not $durations.TryAdd($name, [Math]::Max(1, $duration))) { + throw "Timing history '$Path' contains duplicate test '$name'." + } + } + + return $durations +} + +function New-DurationBalancedWorkItems +{ + param ( + [Parameter(Mandatory)] + [string[]] $TestNames, + + [Parameter(Mandatory)] + [System.Collections.Generic.IDictionary[string,long]] $TimingHistory, + + [Parameter(Mandatory)] + [ValidateRange(1, 1440)] + [int] $TargetMinutes, + + [Parameter(Mandatory)] + [ValidateRange(0.1, 128)] + [double] $DurationParallelism, + + [ValidateRange(1, 86400)] + [int] $FallbackTestDurationSeconds = 60 + ) + + $uniqueNames = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($name in $TestNames) { + if ([string]::IsNullOrWhiteSpace($name)) { + throw 'Test names cannot be empty.' + } + if (-not $uniqueNames.Add($name)) { + throw "Duplicate test name '$name'." + } + } + if ($uniqueNames.Count -eq 0) { + throw 'At least one test is required.' + } + + $knownDurations = [System.Collections.Generic.List[long]]::new() + foreach ($name in $uniqueNames) { + $duration = 0L + if ($TimingHistory.TryGetValue($name, [ref] $duration)) { + $knownDurations.Add($duration) + } + } + + if ($knownDurations.Count -eq 0) { + $missingDuration = [long] $FallbackTestDurationSeconds * 1000 + $durationMode = 'count-fallback' + } else { + $missingDuration = [long] [Math]::Round( + ($knownDurations | Measure-Object -Average).Average, + [MidpointRounding]::AwayFromZero + ) + $durationMode = if ($knownDurations.Count -eq $uniqueNames.Count) { 'historical' } else { 'partial-history' } + } + + $tests = [System.Collections.Generic.List[object]]::new() + foreach ($name in $uniqueNames) { + $duration = 0L + $isEstimated = -not $TimingHistory.TryGetValue($name, [ref] $duration) + if ($isEstimated) { + $duration = $missingDuration + } + $tests.Add([pscustomobject] @{ + Name = $name + DurationMs = $duration + IsEstimated = $isEstimated + }) + } + + $comparison = [System.Comparison[object]] { + param ($left, $right) + $durationComparison = $right.DurationMs.CompareTo($left.DurationMs) + if ($durationComparison -ne 0) { + return $durationComparison + } + return [StringComparer]::Ordinal.Compare($left.Name, $right.Name) + } + $tests.Sort($comparison) + + $targetLoadMs = [long] [Math]::Round( + $TargetMinutes * 60000 * $DurationParallelism, + [MidpointRounding]::AwayFromZero + ) + $bins = [System.Collections.Generic.List[object]]::new() + + foreach ($test in $tests) { + $selectedBin = $null + if ($test.DurationMs -le $targetLoadMs) { + foreach ($bin in $bins) { + if ($bin.Oversized -or $bin.DurationMs + $test.DurationMs -gt $targetLoadMs) { + continue + } + if ($null -eq $selectedBin -or + $bin.DurationMs -gt $selectedBin.DurationMs -or + ($bin.DurationMs -eq $selectedBin.DurationMs -and $bin.Id -lt $selectedBin.Id)) { + $selectedBin = $bin + } + } + } + + if ($null -eq $selectedBin) { + $selectedBin = [pscustomobject] @{ + Id = $bins.Count + 1 + DurationMs = 0L + Oversized = $test.DurationMs -gt $targetLoadMs + Tests = [System.Collections.Generic.List[object]]::new() + } + $bins.Add($selectedBin) + } + + $selectedBin.Tests.Add($test) + $selectedBin.DurationMs += $test.DurationMs + } + + $workItems = [System.Collections.Generic.List[object]]::new() + foreach ($bin in $bins) { + $sortedNames = [System.Collections.Generic.List[string]]::new() + foreach ($test in $bin.Tests) { + $sortedNames.Add($test.Name) + } + $sortedNames.Sort([StringComparer]::Ordinal) + + $workItems.Add([pscustomobject] @{ + Id = $bin.Id + DurationMs = $bin.DurationMs + EstimatedDurationMs = [long] [Math]::Ceiling($bin.DurationMs / $DurationParallelism) + Oversized = $bin.Oversized + Tests = $sortedNames.ToArray() + }) + } + + return [pscustomobject] @{ + DurationMode = $durationMode + KnownTestCount = $knownDurations.Count + EstimatedTestCount = $uniqueNames.Count - $knownDurations.Count + MissingTestDurationMs = $missingDuration + TargetLoadMs = $targetLoadMs + TargetMinutes = $TargetMinutes + DurationParallelism = $DurationParallelism + TotalDurationMs = ($tests | Measure-Object -Property DurationMs -Sum).Sum + WorkItems = $workItems.ToArray() + } +} + +function Write-NUnitRunSettings +{ + param ( + [Parameter(Mandatory)] + [string[]] $TestNames, + + [Parameter(Mandatory)] + [string] $Path + ) + + $settings = [Xml.XmlWriterSettings]::new() + $settings.Indent = $true + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $writer = [Xml.XmlWriter]::Create($Path, $settings) + try { + $filters = foreach ($name in $TestNames) { + $escaped = $name.Replace('\', '\\').Replace("'", "\'") + "test == '$escaped'" + } + $writer.WriteStartElement('RunSettings') + $writer.WriteStartElement('NUnit') + $writer.WriteElementString('Where', ($filters -join ' or ')) + $writer.WriteEndElement() + $writer.WriteEndElement() + } finally { + $writer.Dispose() + } +} + +function Write-HostBuildTestCommand +{ + param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $NUnitWorkers, + + [Parameter(Mandatory)] + [string] $Configuration, + + [Parameter(Mandatory)] + [string] $Path + ) + + if ($Platform -eq 'windows') { + $assemblyPath = $TestAssemblyRelativePath.Replace('/', '\') + $command = @' +@echo off +setlocal +if "%HELIX_CORRELATION_PAYLOAD%"=="" exit /b 2 +if "%HELIX_WORKITEM_ROOT%"=="" exit /b 2 +if "%HELIX_WORKITEM_UPLOAD_ROOT%"=="" exit /b 2 +set "REPO=%HELIX_CORRELATION_PAYLOAD%\repo" +set "DOTNET_ROOT=%REPO%\bin\__CONFIGURATION__\dotnet" +set "ANDROID_HOME=%HELIX_CORRELATION_PAYLOAD%\android-toolchain\sdk" +set "ANDROID_SDK_ROOT=%ANDROID_HOME%" +set "JAVA_HOME=%HELIX_CORRELATION_PAYLOAD%\jdk" +set "NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\nuget-packages" +set "GRADLE_USER_HOME=%HELIX_CORRELATION_PAYLOAD%\gradle" +set "DOTNET_CLI_HOME=%HELIX_WORKITEM_ROOT%\dotnet-home" +set "TEMP=%HELIX_WORKITEM_ROOT%\temp" +set "TMP=%TEMP%" +set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" +set "CONFIGURATION=__CONFIGURATION__" +set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" +set "DOTNET_CLI_TELEMETRY_OPTOUT=1" +set "DOTNET_NOLOGO=1" +set "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" +set "DOTNET_GENERATE_ASPNET_CERTIFICATE=false" +set "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=true" +set "DOTNET_SDK_VULNERABILITY_CHECK_DISABLE=true" +set "DOTNET_SYSTEM_NET_SECURITY_NOREVOCATIONCHECKBYDEFAULT=true" +set "NuGetAudit=false" +set "DOTNET_DbgEnableMiniDump=1" +set "DOTNET_DbgMiniDumpType=2" +set "DOTNET_DbgMiniDumpName=%HELIX_WORKITEM_UPLOAD_ROOT%\dumps\%%e.%%p.dmp" +if not exist "%DOTNET_ROOT%\dotnet.exe" exit /b 3 +if not exist "%REPO%\__TEST_ASSEMBLY__" exit /b 3 +mkdir "%HELIX_WORKITEM_UPLOAD_ROOT%\dumps" 2>nul +mkdir "%DOTNET_CLI_HOME%" 2>nul +mkdir "%NUGET_PACKAGES%" 2>nul +mkdir "%TEMP%" 2>nul +copy /y "%~dp0slice.runsettings" "%HELIX_WORKITEM_UPLOAD_ROOT%\slice.runsettings" >nul +copy /y "%~dp0work-item.json" "%HELIX_WORKITEM_UPLOAD_ROOT%\work-item.json" >nul +pushd "%REPO%" +"%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" 2>&1 +set "testExitCode=%ERRORLEVEL%" +type "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" +"%DOTNET_ROOT%\dotnet.exe" build-server shutdown >nul 2>&1 +popd +exit /b %testExitCode% +'@ + $command = $command.Replace('__TEST_ASSEMBLY__', $assemblyPath) + $command = $command.Replace('__NUNIT_WORKERS__', $NUnitWorkers) + $command = $command.Replace('__CONFIGURATION__', $Configuration) + [IO.File]::WriteAllText($Path, $command, [Text.Encoding]::ASCII) + } else { + $assemblyPath = $TestAssemblyRelativePath.Replace('\', '/') + $command = @' +#!/usr/bin/env bash +set -euo pipefail +: "${HELIX_CORRELATION_PAYLOAD:?}" +: "${HELIX_WORKITEM_ROOT:?}" +: "${HELIX_WORKITEM_UPLOAD_ROOT:?}" +REPO="$HELIX_CORRELATION_PAYLOAD/repo" +export DOTNET_ROOT="$REPO/bin/__CONFIGURATION__/dotnet" +export ANDROID_HOME="$HELIX_CORRELATION_PAYLOAD/android-toolchain/sdk" +export ANDROID_SDK_ROOT="$ANDROID_HOME" +export JAVA_HOME="$HELIX_CORRELATION_PAYLOAD/jdk" +export NUGET_PACKAGES="$HELIX_WORKITEM_ROOT/nuget-packages" +export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" +export DOTNET_CLI_HOME="$HELIX_WORKITEM_ROOT/dotnet-home" +export TMPDIR="$HELIX_WORKITEM_ROOT/temp" +export BUILD_STAGINGDIRECTORY="$HELIX_WORKITEM_UPLOAD_ROOT" +export CONFIGURATION=__CONFIGURATION__ +export PATH="$HELIX_CORRELATION_PAYLOAD/dotnet-tools:$DOTNET_ROOT:$ANDROID_HOME/platform-tools:$JAVA_HOME/bin:$PATH" +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_NOLOGO=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +export DOTNET_GENERATE_ASPNET_CERTIFICATE=false +export DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=true +export DOTNET_SDK_VULNERABILITY_CHECK_DISABLE=true +export DOTNET_SYSTEM_NET_SECURITY_NOREVOCATIONCHECKBYDEFAULT=true +export NuGetAudit=false +export DOTNET_DbgEnableMiniDump=1 +export DOTNET_DbgMiniDumpType=2 +export DOTNET_DbgMiniDumpName="$HELIX_WORKITEM_UPLOAD_ROOT/dumps/%e.%p.dmp" +test -x "$DOTNET_ROOT/dotnet" +test -f "$REPO/__TEST_ASSEMBLY__" +mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$NUGET_PACKAGES" "$TMPDIR" +cp "$HELIX_WORKITEM_ROOT/slice.runsettings" "$HELIX_WORKITEM_UPLOAD_ROOT/slice.runsettings" +cp "$HELIX_WORKITEM_ROOT/work-item.json" "$HELIX_WORKITEM_UPLOAD_ROOT/work-item.json" +cd "$REPO" +set +e +"$DOTNET_ROOT/dotnet" test "$REPO/__TEST_ASSEMBLY__" --settings "$HELIX_WORKITEM_ROOT/slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "$HELIX_WORKITEM_UPLOAD_ROOT" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ 2>&1 | tee "$HELIX_WORKITEM_UPLOAD_ROOT/console.log" +test_exit=${PIPESTATUS[0]} +set -e +"$DOTNET_ROOT/dotnet" build-server shutdown >/dev/null 2>&1 || true +exit "$test_exit" +'@ + $command = $command.Replace('__TEST_ASSEMBLY__', $assemblyPath) + $command = $command.Replace('__NUNIT_WORKERS__', $NUnitWorkers) + $command = $command.Replace('__CONFIGURATION__', $Configuration) + [IO.File]::WriteAllText($Path, $command, [Text.UTF8Encoding]::new($false)) + } +} + +function Write-HostBuildTestWorkItemPayloads +{ + param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [object] $Plan, + + [Parameter(Mandatory)] + [string] $WorkItemsDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsPropsPath, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $NUnitWorkers, + + [Parameter(Mandatory)] + [string] $Configuration + ) + + Remove-Item -LiteralPath $WorkItemsDirectory -Recurse -Force -ErrorAction Ignore + New-Item -ItemType Directory -Force -Path $WorkItemsDirectory | Out-Null + + $propsSettings = [Xml.XmlWriterSettings]::new() + $propsSettings.Indent = $true + $propsSettings.Encoding = [Text.UTF8Encoding]::new($false) + $propsWriter = [Xml.XmlWriter]::Create($WorkItemsPropsPath, $propsSettings) + try { + $propsWriter.WriteStartElement('Project') + $propsWriter.WriteStartElement('ItemGroup') + + foreach ($workItem in $Plan.WorkItems) { + $name = 'host-build-tests-{0}-{1:d3}' -f $Platform, $workItem.Id + $payloadDirectory = Join-Path $WorkItemsDirectory $name + New-Item -ItemType Directory -Force -Path $payloadDirectory | Out-Null + + Write-NUnitRunSettings -TestNames $workItem.Tests -Path (Join-Path $payloadDirectory 'slice.runsettings') + $commandName = if ($Platform -eq 'windows') { 'run-host-tests.cmd' } else { 'run-host-tests.sh' } + $helixCommand = if ($Platform -eq 'windows') { $commandName } else { "bash $commandName" } + Write-HostBuildTestCommand ` + -Platform $Platform ` + -TestAssemblyRelativePath $TestAssemblyRelativePath ` + -NUnitWorkers $NUnitWorkers ` + -Configuration $Configuration ` + -Path (Join-Path $payloadDirectory $commandName) + + [pscustomobject] @{ + name = $name + durationMode = $Plan.DurationMode + estimatedDurationMs = $workItem.EstimatedDurationMs + durationLoadMs = $workItem.DurationMs + oversized = $workItem.Oversized + targetMinutes = $Plan.TargetMinutes + durationParallelism = $Plan.DurationParallelism + testCount = $workItem.Tests.Count + tests = $workItem.Tests + } | ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath (Join-Path $payloadDirectory 'work-item.json') -Encoding utf8NoBOM + + $propsWriter.WriteStartElement('_HostBuildTestHelixWorkItem') + $propsWriter.WriteAttributeString('Include', $name) + $propsWriter.WriteElementString('PayloadDirectory', $payloadDirectory) + $propsWriter.WriteElementString('Command', $helixCommand) + $propsWriter.WriteElementString('EstimatedDurationMs', [string] $workItem.EstimatedDurationMs) + $propsWriter.WriteElementString('TestCount', [string] $workItem.Tests.Count) + $propsWriter.WriteEndElement() + } + + $propsWriter.WriteEndElement() + $propsWriter.WriteEndElement() + } finally { + $propsWriter.Dispose() + } +} + +Export-ModuleMember -Function ` + Get-NUnitRunSettingsTestNames, ` + Get-HostBuildTestTimingHistory, ` + New-DurationBalancedWorkItems, ` + Write-NUnitRunSettings, ` + Write-HostBuildTestWorkItemPayloads diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md new file mode 100644 index 00000000000..476dcc348d4 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -0,0 +1,29 @@ +# Host build tests in Helix prototype + +Tracking: [AB#12334](https://dev.azure.com/dnceng/internal/_workitems/edit/12334) + +This guarded prototype stages the already-prepared .NET SDK/workload, Android SDK, +JDK, selected Gradle caches, repository inputs, and test assemblies as shared Helix +correlation payloads. Individual payloads are capped at 1 GiB to avoid the Helix SDK's +in-memory ZIP limit. Work item payloads contain only a runsettings filter, a small +command script, and generation metadata. To control total payload size, the source +copy omits `src/Mono.Android` and the NuGet package cache; host build tests consume the +product through the prepared workload packs and use isolated per-work-item NuGet +caches. Device-only Android SDK content, SDK documentation, the Java.Interop checkout, +and unrelated test assemblies are also excluded. + +`prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's +`dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit +decreasing bin packing. The target is `15` estimated wall-clock minutes by default. +`durationParallelism` translates summed per-test elapsed time into expected wall time +while preserving the current NUnit worker setting. The public prototype defaults this +factor to `2.5`, uses `60` seconds per test for the explicit first-run fallback, and +allows `45` minutes before Helix terminates a work item. + +If no timing history is provided, generation explicitly reports `count-fallback` and +uses the configured fallback duration per test. It does not label count-only slicing +as duration-balanced. Each completed run publishes a new `balance.xml`; pass that +run's build ID through `hostBuildTestsHelixTimingBuildId` on the next queued build. + +The public pipeline keeps the existing Azure Pipelines jobs and adds this path only +when `enableHostBuildTestsHelixPrototype` is `true`. diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 new file mode 100644 index 00000000000..7dc5d590fdb --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -0,0 +1,466 @@ +[CmdletBinding()] +param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [string] $RepositoryRoot, + + [Parameter(Mandatory)] + [string] $CorrelationPayloadDirectory, + + [Parameter(Mandatory)] + [string] $CorrelationPayloadsPropsPath, + + [Parameter(Mandatory)] + [string] $WorkItemsDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsPropsPath, + + [Parameter(Mandatory)] + [string] $ResultsDirectory, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $TestSlicerPath, + + [string] $TestFilter = '', + + [string] $BalanceFile = '', + + [ValidateRange(1, 1440)] + [int] $TargetMinutes = 15, + + [ValidateRange(0.1, 128)] + [double] $DurationParallelism = 2.5, + + [ValidateRange(1, 86400)] + [int] $FallbackTestDurationSeconds = 60, + + [string] $NUnitWorkers = '-1', + + [string] $Configuration = 'Release', + + [Parameter(Mandatory)] + [string] $AndroidHome, + + [Parameter(Mandatory)] + [string] $JavaHome, + + [string] $DotNetToolsDirectory = '', + + [string] $GradleUserHome = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'HostBuildTestsHelix.psm1') -Force + +function ConvertTo-NativeRelativePath +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $separator = [string] [IO.Path]::DirectorySeparatorChar + return $Path.Replace('\', $separator).Replace('/', $separator) +} + +function Copy-PayloadDirectory +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [switch] $Optional + ) + + if (-not (Test-Path -LiteralPath $Source -PathType Container)) { + if ($Optional) { + Write-Warning "Optional payload directory '$Source' does not exist." + return + } + throw "Required payload directory '$Source' does not exist." + } + + Write-Host "Staging payload directory '$Source'." + New-Item -ItemType Directory -Force -Path $Destination | Out-Null + if ($IsWindows) { + & robocopy $Source $Destination /E /COPY:DAT /DCOPY:DAT /R:2 /W:1 /NFL /NDL /NJH /NJS /NP + if ($LASTEXITCODE -gt 7) { + throw "robocopy failed with exit code $LASTEXITCODE while copying '$Source'." + } + } else { + & cp -a "$Source/." "$Destination/" + if ($LASTEXITCODE -ne 0) { + throw "cp failed with exit code $LASTEXITCODE while copying '$Source'." + } + } +} + +function Get-DirectorySize +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $size = 0L + foreach ($file in Get-ChildItem -LiteralPath $Path -Recurse -File) { + $size += $file.Length + } + return $size +} + +$correlationPayloads = [System.Collections.Generic.List[object]]::new() +$filePayloadIndex = 0 +$maximumPayloadBytes = 1GB + +function Add-CorrelationPayload +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [Parameter(Mandatory)] + [long] $SizeBytes + ) + + $script:correlationPayloads.Add([pscustomobject] @{ + Source = [IO.Path]::GetFullPath($Source) + Destination = $Destination.Replace('\', '/').Trim('/') + SizeBytes = $SizeBytes + }) +} + +function Add-FileCorrelationPayloads +{ + param ( + [Parameter(Mandatory)] + [IO.FileInfo[]] $Files, + + [Parameter(Mandatory)] + [string] $Destination + ) + + $currentFiles = [System.Collections.Generic.List[IO.FileInfo]]::new() + $currentSize = 0L + foreach ($file in $Files | Sort-Object Name) { + if ($file.Length -gt $script:maximumPayloadBytes) { + throw "File '$($file.FullName)' exceeds the maximum correlation payload size." + } + if ($currentFiles.Count -gt 0 -and $currentSize + $file.Length -gt $script:maximumPayloadBytes) { + New-FileCorrelationPayload -Files $currentFiles.ToArray() -Destination $Destination -SizeBytes $currentSize + $currentFiles.Clear() + $currentSize = 0L + } + $currentFiles.Add($file) + $currentSize += $file.Length + } + if ($currentFiles.Count -gt 0) { + New-FileCorrelationPayload -Files $currentFiles.ToArray() -Destination $Destination -SizeBytes $currentSize + } +} + +function New-FileCorrelationPayload +{ + param ( + [Parameter(Mandatory)] + [IO.FileInfo[]] $Files, + + [Parameter(Mandatory)] + [string] $Destination, + + [Parameter(Mandatory)] + [long] $SizeBytes + ) + + $script:filePayloadIndex++ + $payloadDirectory = Join-Path $script:correlationPayloadFullPath ('files-{0:d3}' -f $script:filePayloadIndex) + New-Item -ItemType Directory -Force -Path $payloadDirectory | Out-Null + foreach ($file in $Files) { + $target = Join-Path $payloadDirectory $file.Name + try { + New-Item -ItemType HardLink -Path $target -Target $file.FullName | Out-Null + } catch { + Copy-Item -LiteralPath $file.FullName -Destination $target + } + } + Add-CorrelationPayload -Source $payloadDirectory -Destination $Destination -SizeBytes $SizeBytes +} + +function Add-DirectoryCorrelationPayloads +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [switch] $Optional + ) + + if (-not (Test-Path -LiteralPath $Source -PathType Container)) { + if ($Optional) { + Write-Warning "Optional correlation payload directory '$Source' does not exist." + return + } + throw "Required correlation payload directory '$Source' does not exist." + } + + $size = Get-DirectorySize -Path $Source + if ($size -le $script:maximumPayloadBytes) { + Add-CorrelationPayload -Source $Source -Destination $Destination -SizeBytes $size + return + } + + $files = @(Get-ChildItem -LiteralPath $Source -File) + if ($files.Count -gt 0) { + Add-FileCorrelationPayloads -Files $files -Destination $Destination + } + foreach ($directory in Get-ChildItem -LiteralPath $Source -Directory | Sort-Object Name) { + Add-DirectoryCorrelationPayloads ` + -Source $directory.FullName ` + -Destination "$($Destination.TrimEnd('/'))/$($directory.Name)" + } +} + +function Write-CorrelationPayloadProps +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $settings = [Xml.XmlWriterSettings]::new() + $settings.Indent = $true + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $writer = [Xml.XmlWriter]::Create($Path, $settings) + try { + $writer.WriteStartElement('Project') + $writer.WriteStartElement('ItemGroup') + foreach ($payload in $script:correlationPayloads) { + $writer.WriteStartElement('_HostBuildTestHelixCorrelationPayload') + $writer.WriteAttributeString('Include', $payload.Source) + $writer.WriteElementString('Destination', $payload.Destination) + $writer.WriteEndElement() + } + $writer.WriteEndElement() + $writer.WriteEndElement() + } finally { + $writer.Dispose() + } +} + +$repositoryRootFullPath = [IO.Path]::GetFullPath($RepositoryRoot) +$correlationPayloadFullPath = [IO.Path]::GetFullPath($CorrelationPayloadDirectory) +$correlationPayloadsPropsFullPath = [IO.Path]::GetFullPath($CorrelationPayloadsPropsPath) +$workItemsFullPath = [IO.Path]::GetFullPath($WorkItemsDirectory) +$workItemsPropsFullPath = [IO.Path]::GetFullPath($WorkItemsPropsPath) +$resultsFullPath = [IO.Path]::GetFullPath($ResultsDirectory) + +Remove-Item -LiteralPath $correlationPayloadFullPath -Recurse -Force -ErrorAction Ignore +Remove-Item -LiteralPath $workItemsFullPath -Recurse -Force -ErrorAction Ignore +New-Item -ItemType Directory -Force -Path $correlationPayloadFullPath, $workItemsFullPath, $resultsFullPath | Out-Null + +$payloadRepository = Join-Path $correlationPayloadFullPath 'repository' +New-Item -ItemType Directory -Force -Path $payloadRepository | Out-Null +Get-ChildItem -LiteralPath $repositoryRootFullPath -File | Copy-Item -Destination $payloadRepository + +$payloadDirectories = @( + @('build-tools', 'build-tools', $false), + @('eng', 'eng', $false), + @('.github', '.github', $true), + @('samples', 'samples', $true), + @('src-ThirdParty', 'src-ThirdParty', $true), + @('tests', 'tests', $false), + @('tools', 'tools', $true) +) +foreach ($directory in $payloadDirectories) { + Copy-PayloadDirectory ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $directory[0])) ` + -Destination (Join-Path $payloadRepository (ConvertTo-NativeRelativePath $directory[1])) ` + -Optional:([bool] $directory[2]) +} + +$sourceRoot = Join-Path $repositoryRootFullPath 'src' +foreach ($sourceDirectory in Get-ChildItem -LiteralPath $sourceRoot -Directory | Where-Object Name -ne 'Mono.Android') { + Copy-PayloadDirectory ` + -Source $sourceDirectory.FullName ` + -Destination (Join-Path (Join-Path $payloadRepository 'src') $sourceDirectory.Name) +} + +$payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' +if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { + New-Item -ItemType Directory -Force -Path $payloadTools | Out-Null + $apkDiffShim = if ($Platform -eq 'windows') { 'apkdiff.exe' } else { 'apkdiff' } + $apkDiffShimPath = Join-Path $DotNetToolsDirectory $apkDiffShim + if (Test-Path -LiteralPath $apkDiffShimPath -PathType Leaf) { + Copy-Item -LiteralPath $apkDiffShimPath -Destination $payloadTools + } + Copy-PayloadDirectory ` + -Source (Join-Path $DotNetToolsDirectory (ConvertTo-NativeRelativePath '.store/apkdiff')) ` + -Destination (Join-Path $payloadTools (ConvertTo-NativeRelativePath '.store/apkdiff')) ` + -Optional +} + +Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` + -Destination "repo/bin/$Configuration" + +$testOutputRoot = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration") +$testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) +$testAssemblyDirectory = Split-Path -Parent $testAssembly +$testAssemblyDestination = "repo/bin/Test$Configuration/$([IO.Path]::GetRelativePath($testOutputRoot, $testAssemblyDirectory).Replace('\', '/'))" +Add-DirectoryCorrelationPayloads -Source $testAssemblyDirectory -Destination $testAssemblyDestination +$testOutputFiles = @(Get-ChildItem -LiteralPath $testOutputRoot -File) +if ($testOutputFiles.Count -gt 0) { + Add-FileCorrelationPayloads -Files $testOutputFiles -Destination "repo/bin/Test$Configuration" +} +foreach ($testDirectory in Get-ChildItem -LiteralPath $testOutputRoot -Directory | + Where-Object { $_.Name -eq 'Expected' -or $_.Name -like 'android-*' }) { + Add-DirectoryCorrelationPayloads ` + -Source $testDirectory.FullName ` + -Destination "repo/bin/Test$Configuration/$($testDirectory.Name)" +} + +$androidRootFiles = @(Get-ChildItem -LiteralPath $AndroidHome -File) +if ($androidRootFiles.Count -gt 0) { + Add-FileCorrelationPayloads -Files $androidRootFiles -Destination 'android-toolchain/sdk' +} +$excludedAndroidDirectories = @('docs', 'emulator', 'extras', 'skins', 'sources', 'system-images') +foreach ($androidDirectory in Get-ChildItem -LiteralPath $AndroidHome -Directory | Sort-Object Name) { + if ($androidDirectory.Name -notin $excludedAndroidDirectories) { + Add-DirectoryCorrelationPayloads ` + -Source $androidDirectory.FullName ` + -Destination "android-toolchain/sdk/$($androidDirectory.Name)" + } +} + +Add-DirectoryCorrelationPayloads -Source $JavaHome -Destination 'jdk' + +if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { + $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' +} +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'wrapper/dists')) ` + -Destination 'gradle/wrapper/dists' ` + -Optional +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'caches/modules-2')) ` + -Destination 'gradle/caches/modules-2' ` + -Optional +if (Test-Path -LiteralPath $payloadTools -PathType Container) { + Add-DirectoryCorrelationPayloads -Source $payloadTools -Destination 'dotnet-tools' +} + +Write-CorrelationPayloadProps -Path $correlationPayloadsPropsFullPath +$payloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Sum).Sum +$largestPayloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Maximum).Maximum +Write-Host ('Prepared {0} correlation payloads totaling {1:N2} GiB; largest payload is {2:N2} GiB.' -f ` + $correlationPayloads.Count, ($payloadBytes / 1GB), ($largestPayloadBytes / 1GB)) +Write-Host 'Largest correlation payload components:' +$correlationPayloads | + Sort-Object SizeBytes -Descending | + Select-Object -First 20 | + ForEach-Object { + Write-Host ('- {0:N2} GiB -> {1}' -f ($_.SizeBytes / 1GB), $_.Destination) + } + +if (-not (Test-Path -LiteralPath $testAssembly -PathType Leaf)) { + throw "Test assembly '$testAssembly' does not exist." +} +if (-not (Test-Path -LiteralPath $TestSlicerPath -PathType Leaf)) { + throw "dotnet-test-slicer '$TestSlicerPath' does not exist." +} + +$discoveryRunSettings = Join-Path $workItemsFullPath 'all-tests.runsettings' +$sliceArguments = @( + 'slice', + "--test-assembly=$testAssembly", + '--slice-number=1', + '--total-slices=1', + "--outfile=$discoveryRunSettings" +) +if (-not [string]::IsNullOrWhiteSpace($TestFilter)) { + $sliceArguments += "--test-filter=$TestFilter" +} + +& $TestSlicerPath @sliceArguments +if ($LASTEXITCODE -ne 0) { + throw "dotnet-test-slicer discovery failed with exit code $LASTEXITCODE." +} + +$testNames = Get-NUnitRunSettingsTestNames -Path $discoveryRunSettings +$timingHistory = Get-HostBuildTestTimingHistory -Path $BalanceFile +$plan = New-DurationBalancedWorkItems ` + -TestNames $testNames ` + -TimingHistory $timingHistory ` + -TargetMinutes $TargetMinutes ` + -DurationParallelism $DurationParallelism ` + -FallbackTestDurationSeconds $FallbackTestDurationSeconds + +Write-HostBuildTestWorkItemPayloads ` + -Platform $Platform ` + -Plan $plan ` + -WorkItemsDirectory $workItemsFullPath ` + -WorkItemsPropsPath $workItemsPropsFullPath ` + -TestAssemblyRelativePath $TestAssemblyRelativePath ` + -NUnitWorkers $NUnitWorkers ` + -Configuration $Configuration + +$summaryPath = Join-Path $resultsFullPath 'work-item-generation.json' +[pscustomobject] @{ + platform = $Platform + testFilter = $TestFilter + durationMode = $plan.DurationMode + targetMinutes = $TargetMinutes + durationParallelism = $DurationParallelism + fallbackTestDurationSeconds = $FallbackTestDurationSeconds + testCount = $testNames.Count + knownTestCount = $plan.KnownTestCount + estimatedTestCount = $plan.EstimatedTestCount + workItemCount = $plan.WorkItems.Count + totalDurationLoadMs = $plan.TotalDurationMs + correlationPayloadCount = $correlationPayloads.Count + correlationPayloadBytes = $payloadBytes + correlationPayloads = @($correlationPayloads | ForEach-Object { + [pscustomobject] @{ + destination = $_.Destination + sizeBytes = $_.SizeBytes + } + }) + workItems = @($plan.WorkItems | ForEach-Object { + [pscustomobject] @{ + id = $_.Id + testCount = $_.Tests.Count + durationLoadMs = $_.DurationMs + estimatedDurationMs = $_.EstimatedDurationMs + oversized = $_.Oversized + } + }) +} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $summaryPath -Encoding utf8NoBOM + +Write-Host "Generated $($plan.WorkItems.Count) $Platform Helix work items for $($testNames.Count) tests." +Write-Host "Duration mode: $($plan.DurationMode); known timings: $($plan.KnownTestCount); estimated timings: $($plan.EstimatedTestCount)." +Write-Host "Target: $TargetMinutes minute(s); duration parallelism: $DurationParallelism." +if ($plan.DurationMode -eq 'count-fallback') { + Write-Warning "No matching timing history was available. Work items use an explicit count fallback of $FallbackTestDurationSeconds second(s) per test." +} + +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixCorrelationPayloadsProps]$correlationPayloadsPropsFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemsProps]$workItemsPropsFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemCount]$($plan.WorkItems.Count)" diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml new file mode 100644 index 00000000000..e6c95925e8a --- /dev/null +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -0,0 +1,177 @@ +parameters: + platform: '' + helixQueue: '' + testFilter: '' + testRunTitle: '' + timingBuildId: '' + timingArtifactName: '' + targetMinutes: 15 + durationParallelism: 2.5 + fallbackTestDurationSeconds: 60 + workItemTimeout: '00:45:00' + nunitWorkers: '-1' + +steps: +- template: /build-tools/automation/yaml-templates/setup-test-environment-public.yaml + parameters: + useAgentJdkPath: false + installTestSlicer: true + use1ESTemplate: false + +- template: /build-tools/automation/yaml-templates/cache-gradle.yaml + parameters: + xaSourcePath: $(System.DefaultWorkingDirectory) + +- task: DownloadPipelineArtifact@2 + displayName: Download Xamarin.Android.Build.Tests assets + inputs: + artifactName: $(TestAssembliesArtifactName) + downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) + +- ${{ if ne(parameters.timingBuildId, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download prior ${{ parameters.platform }} host-test timings + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + buildVersionToDownload: specific + pipelineId: ${{ parameters.timingBuildId }} + artifactName: ${{ parameters.timingArtifactName }} + targetPath: $(Pipeline.Workspace)/host-build-test-history-${{ parameters.platform }} + +- pwsh: | + $ErrorActionPreference = 'Stop' + $platform = '${{ parameters.platform }}' + $historyRoot = '$(Pipeline.Workspace)/host-build-test-history-${{ parameters.platform }}' + $balanceFile = '' + if ('${{ parameters.timingBuildId }}') { + $balanceFile = Get-ChildItem -LiteralPath $historyRoot -Recurse -Filter '*.xml' | + Select-Object -First 1 -ExpandProperty FullName + if (-not $balanceFile) { + throw "Build '${{ parameters.timingBuildId }}' did not provide timing XML in '$historyRoot'." + } + } + + $testSlicer = Join-Path '$(Agent.ToolsDirectory)' $(if ($platform -eq 'windows') { 'dotnet-test-slicer.exe' } else { 'dotnet-test-slicer' }) + $stagingRoot = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}' + $resultsDirectory = Join-Path $stagingRoot 'results' + $correlationPayload = Join-Path $stagingRoot 'correlation' + $correlationPayloadsProps = Join-Path $stagingRoot 'correlation-payloads.props' + $workItemsDirectory = Join-Path $stagingRoot 'work-items' + $workItemsProps = Join-Path $stagingRoot 'work-items.props' + $adbName = if ($platform -eq 'windows') { 'adb.exe' } else { 'adb' } + $androidHome = @( + $env:ANDROID_HOME + $env:ANDROID_SDK_ROOT + $env:AndroidSdkDirectory + $(if ($env:USERPROFILE) { Join-Path $env:USERPROFILE 'android-toolchain\sdk' }) + $(if ($env:HOME) { Join-Path $env:HOME 'android-toolchain/sdk' }) + ) | Where-Object { $_ -and (Test-Path (Join-Path $_ "platform-tools/$adbName")) } | Select-Object -First 1 + if (-not $androidHome) { + throw 'Could not locate the prepared Android SDK.' + } + + & '$(System.DefaultWorkingDirectory)/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1' ` + -Platform $platform ` + -RepositoryRoot '$(System.DefaultWorkingDirectory)' ` + -CorrelationPayloadDirectory $correlationPayload ` + -CorrelationPayloadsPropsPath $correlationPayloadsProps ` + -WorkItemsDirectory $workItemsDirectory ` + -WorkItemsPropsPath $workItemsProps ` + -ResultsDirectory $resultsDirectory ` + -TestAssemblyRelativePath 'bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll' ` + -TestSlicerPath $testSlicer ` + -TestFilter '${{ parameters.testFilter }}' ` + -BalanceFile $balanceFile ` + -TargetMinutes ${{ parameters.targetMinutes }} ` + -DurationParallelism ${{ parameters.durationParallelism }} ` + -FallbackTestDurationSeconds ${{ parameters.fallbackTestDurationSeconds }} ` + -NUnitWorkers '${{ parameters.nunitWorkers }}' ` + -Configuration '$(XA.Build.Configuration)' ` + -AndroidHome $androidHome ` + -JavaHome $env:JAVA_HOME ` + -DotNetToolsDirectory '$(Agent.ToolsDirectory)' ` + -GradleUserHome (Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle') + displayName: Prepare duration-balanced ${{ parameters.platform }} Helix work items + +- template: /build-tools/automation/yaml-templates/run-dotnet-preview.yaml + parameters: + command: msbuild + project: $(System.DefaultWorkingDirectory)/build-tools/automation/HostBuildTestsHelix.proj + arguments: >- + /restore + /t:Test + /p:TreatWarningsAsErrors=false + /bl:$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results/send-to-helix.binlog + displayName: Submit ${{ parameters.platform }} host build tests to Helix + continueOnError: false + taskTimeoutInMinutes: 240 + env: + HostBuildTestsHelixWorkItemsProps: $(HostBuildTestsHelixWorkItemsProps) + HostBuildTestsHelixCorrelationPayloadsProps: $(HostBuildTestsHelixCorrelationPayloadsProps) + HostBuildTestsHelixWorkItemTimeout: ${{ parameters.workItemTimeout }} + HelixResultsDestinationDir: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results + HelixSource: pr/dotnet/android + HelixType: tests/android/host-build-tests/${{ parameters.platform }}/ + HelixBuild: $(Build.BuildNumber) + HelixConfiguration: ${{ parameters.platform }} + HelixTargetQueueForItem: ${{ parameters.helixQueue }} + HelixTestRunName: ${{ parameters.testRunTitle }} + HelixAccessToken: '' + WaitForWorkItemCompletion: 'true' + Creator: dotnet-android + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +- task: PublishTestResults@2 + displayName: Publish ${{ parameters.platform }} Helix TRX results + condition: always() + inputs: + testResultsFormat: VSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results + mergeTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + continueOnError: true + +- pwsh: | + $ErrorActionPreference = 'Stop' + $resultsDirectory = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results' + $timingDirectory = '$(Build.StagingDirectory)/host-build-test-timings-${{ parameters.platform }}' + $trxDirectory = Join-Path $timingDirectory 'trx' + New-Item -ItemType Directory -Force -Path $trxDirectory | Out-Null + + $trxFiles = @(Get-ChildItem -LiteralPath $resultsDirectory -Recurse -Filter '*.trx' -ErrorAction Ignore) + if ($trxFiles.Count -eq 0) { + Write-Warning "No TRX files were available to produce next-run timings." + Write-Host '##vso[task.setvariable variable=HostBuildTestsHelixHasTimings]false' + exit 0 + } + + for ($index = 0; $index -lt $trxFiles.Count; $index++) { + Copy-Item -LiteralPath $trxFiles[$index].FullName -Destination (Join-Path $trxDirectory ('{0:d4}.trx' -f $index)) + } + + $testSlicer = Join-Path '$(Agent.ToolsDirectory)' $(if ('${{ parameters.platform }}' -eq 'windows') { 'dotnet-test-slicer.exe' } else { 'dotnet-test-slicer' }) + & $testSlicer balance --input-directory=$trxDirectory --outfile=(Join-Path $timingDirectory 'balance.xml') + if ($LASTEXITCODE -ne 0) { + throw "dotnet-test-slicer balance failed with exit code $LASTEXITCODE." + } + Remove-Item -LiteralPath $trxDirectory -Recurse -Force + Write-Host '##vso[task.setvariable variable=HostBuildTestsHelixHasTimings]true' + displayName: Produce next-run ${{ parameters.platform }} timings + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.platform }} host-test timings + condition: and(always(), eq(variables['HostBuildTestsHelixHasTimings'], 'true')) + inputs: + artifactName: ${{ parameters.timingArtifactName }} + targetPath: $(Build.StagingDirectory)/host-build-test-timings-${{ parameters.platform }} + +- task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.platform }} Helix diagnostics + condition: always() + inputs: + artifactName: Host Build Test Helix Results - ${{ parameters.platform }} + targetPath: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results