From 0582a55b90786b4f4d2e94d282393c42113423c6 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Wed, 6 May 2026 03:21:20 -0500 Subject: [PATCH 1/2] Fix dotnet restore hang during prepare step (#11282) The `dotnet restore` of `package-download.proj` was hanging during the prepare phase, causing all 3 retry attempts to time out after 10 minutes each with zero stdout output. Two issues fixed: 1. Double-quoting of arguments: `ProcessRunner.QuoteArgument()` was called before passing args to the `ProcessRunner` constructor, but the constructor already quotes all arguments via `AddQuotedArgument()`. This caused paths to be wrapped in double quotes (e.g. `"\"/path/to/file\""`), which could cause `dotnet restore` to hang trying to resolve an invalid path. 2. Added `DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true` to the shared pipeline variables so all CI pipelines skip the .NET first-run experience, which can also hang on CI agents. Fixes: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=14008410 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/yaml-templates/variables.yaml | 4 +++ .../Steps/Step_InstallDotNetPreview.cs | 36 ++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/build-tools/automation/yaml-templates/variables.yaml b/build-tools/automation/yaml-templates/variables.yaml index 27bcf5d0ca5..925d17b4996 100644 --- a/build-tools/automation/yaml-templates/variables.yaml +++ b/build-tools/automation/yaml-templates/variables.yaml @@ -74,3 +74,7 @@ variables: value: true - name: DOTNET_SYSTEM_NET_SECURITY_NOREVOCATIONCHECKBYDEFAULT value: true +- name: DOTNET_CLI_TELEMETRY_OPTOUT + value: true +- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: true diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs index 4323ff2f8fc..c5896a83c69 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs @@ -41,14 +41,34 @@ protected override async Task Execute (Context context) // Install runtime packs associated with the SDK previously installed. var packageDownloadProj = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "build-tools", "xaprepare", "xaprepare", "package-download.proj"); - var logPath = Path.Combine (Configurables.Paths.BuildBinDir, $"msbuild-{context.BuildTimeStamp}-download-runtime-packs.binlog"); - var restoreArgs = new string [] { "restore", - ProcessRunner.QuoteArgument (packageDownloadProj), - "--configfile", Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "NuGet.config"), - ProcessRunner.QuoteArgument ($"-bl:{logPath}"), - }; - if (!Utilities.RunCommand (Configurables.Paths.DotNetPreviewTool, restoreArgs)) { - Log.ErrorLine ($"Failed to restore runtime packs using '{packageDownloadProj}'."); + var logPathBase = Path.Combine (Configurables.Paths.BuildBinDir, $"msbuild-{context.BuildTimeStamp}-download-runtime-packs"); + + const int maxAttempts = 3; + const int initialBackoffDelayMilliseconds = 2000; + bool restoreSucceeded = false; + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + var logPath = $"{logPathBase}-attempt{attempt}.binlog"; + var runner = new ProcessRunner (Configurables.Paths.DotNetPreviewTool, "restore", + packageDownloadProj, + "--configfile", Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "NuGet.config"), + $"-bl:{logPath}", + "--verbosity", "normal" + ) { + EchoStandardOutput = true, + EchoStandardError = true, + }; + if (runner.Run ()) { + restoreSucceeded = true; + break; + } + if (attempt < maxAttempts) { + Log.WarningLine ($"Failed to restore runtime packs (attempt {attempt}/{maxAttempts}), retrying..."); + var delayMilliseconds = initialBackoffDelayMilliseconds * (1 << (attempt - 1)); + await Task.Delay (delayMilliseconds); + } + } + if (!restoreSucceeded) { + Log.ErrorLine ($"Failed to restore runtime packs using '{packageDownloadProj}' after {maxAttempts} attempts."); return false; } From 9ebde1688b0291364a1b5b0c5a39563b6d69f4e1 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Fri, 8 May 2026 18:40:01 -0500 Subject: [PATCH 2/2] Fix dotnet restore hang in Step_InstallDotNetPreview (#11310) * Increase ProcessTimeout for runtime pack restore to 30 minutes The dotnet restore of package-download.proj downloads ~18 NuGet runtime packs and workload manifests. The default ProcessRunner timeout of 10 minutes is not enough on slower CI machines, causing all 3 retry attempts to time out and the entire Step_InstallDotNetPreview to fail. Increase the ProcessTimeout to 30 minutes for this specific restore. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix dotnet restore hang in Step_InstallDotNetPreview Three changes to address the recurring restore hang on macOS CI: 1. Kill entire process tree on timeout: process.Kill() only kills the top-level dotnet process, leaving orphaned MSBuild worker nodes alive. These can hold locks that block subsequent retry attempts. Use process.Kill(entireProcessTree: true) instead. 2. Replace DOTNET_SKIP_FIRST_TIME_EXPERIENCE (ignored since .NET 5) with DOTNET_NOLOGO and DOTNET_GENERATE_ASPNET_CERTIFICATE=false. The first-run experience still runs ASP.NET cert generation on macOS which involves keychain operations that can hang on CI. 3. Revert the 30-minute ProcessTimeout since 10 minutes is already generous for a restore that should take seconds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/yaml-templates/variables.yaml | 4 +++- build-tools/xaprepare/xaprepare/Application/ProcessRunner.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build-tools/automation/yaml-templates/variables.yaml b/build-tools/automation/yaml-templates/variables.yaml index 925d17b4996..1bbe24740be 100644 --- a/build-tools/automation/yaml-templates/variables.yaml +++ b/build-tools/automation/yaml-templates/variables.yaml @@ -76,5 +76,7 @@ variables: value: true - name: DOTNET_CLI_TELEMETRY_OPTOUT value: true -- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE +- name: DOTNET_NOLOGO value: true +- name: DOTNET_GENERATE_ASPNET_CERTIFICATE + value: false diff --git a/build-tools/xaprepare/xaprepare/Application/ProcessRunner.cs b/build-tools/xaprepare/xaprepare/Application/ProcessRunner.cs index b225ddd897a..db4f11f7675 100644 --- a/build-tools/xaprepare/xaprepare/Application/ProcessRunner.cs +++ b/build-tools/xaprepare/xaprepare/Application/ProcessRunner.cs @@ -290,7 +290,7 @@ public bool Run () if (!exited) { Log.ErrorLine ($"Process '{FullCommandLine}' timed out after {ProcessTimeout}"); ErrorReason = ErrorReasonCode.ExecutionTimedOut; - process.Kill (); + process.Kill (entireProcessTree: true); } // See: https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.process.waitforexit?view=netframework-4.7.2#System_Diagnostics_Process_WaitForExit)