Fix CI flakiness: MSB4216 task host failures, dotnet-watch hangs, NuGet errors

build/RunTestsOnHelix.cmd

@@ -9,6 +9,10 @@ set DOTNET_ROOT=%HELIX_CORRELATION_PAYLOAD%\d
 set PATH=%DOTNET_ROOT%;%PATH%
 set TestFullMSBuild=%1
 
+REM Set DOTNET_HOST_PATH so MSBuild task hosts can locate the dotnet executable.
+REM Without this, tasks from NuGet packages that use TaskHostFactory fail with MSB4216.
+set DOTNET_HOST_PATH=%DOTNET_ROOT%\dotnet.exe
+
 REM Ensure Visual Studio instances allow preview SDKs
 PowerShell -ExecutionPolicy ByPass -NoProfile -File "%HELIX_CORRELATION_PAYLOAD%\t\eng\enable-preview-sdks.ps1"
 
@@ -35,14 +39,16 @@ dotnet new --debug:ephemeral-hive
 dotnet nuget list source --configfile %TestExecutionDirectory%\nuget.config
 if exist %TestExecutionDirectory%\Testpackages dotnet nuget add source %TestExecutionDirectory%\Testpackages --name testpackages --configfile %TestExecutionDirectory%\nuget.config
 
-dotnet nuget remove source dotnet6-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet6-internal-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet7-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet7-internal-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source richnav --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source vs-impl --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-libraries-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-tools-transport --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-libraries --configfile %TestExecutionDirectory%\nuget.config
-dotnet nuget remove source dotnet-eng --configfile %TestExecutionDirectory%\nuget.config
+REM Remove feeds not needed for tests. Errors from non-existent sources
+REM (e.g. internal-transport feeds only present in internal builds) are ignored.
+dotnet nuget remove source dotnet6-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet6-internal-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet7-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet7-internal-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source richnav --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source vs-impl --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-libraries-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-tools-transport --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-libraries --configfile %TestExecutionDirectory%\nuget.config 2>nul
+dotnet nuget remove source dotnet-eng --configfile %TestExecutionDirectory%\nuget.config 2>nul
 dotnet nuget list source --configfile %TestExecutionDirectory%\nuget.config

build/RunTestsOnHelix.sh

@@ -9,6 +9,12 @@ export MicrosoftNETBuildExtensionsTargets=$HELIX_CORRELATION_PAYLOAD/ex/msbuildE
 export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/d
 export PATH=$DOTNET_ROOT:$PATH
 
+# Set DOTNET_HOST_PATH so MSBuild task hosts can locate the dotnet executable.
+# Without this, tasks from NuGet packages that use TaskHostFactory (e.g. ComputeWasmBuildAssets
+# from WebAssembly SDK, ComputeManagedAssemblies from ILLink) fail with MSB4216 on macOS
+# because the task host process cannot find the dotnet host to launch.
+export DOTNET_HOST_PATH=$DOTNET_ROOT/dotnet
+
 export TestExecutionDirectory=$(realpath "$(mktemp -d "${TMPDIR:-/tmp}"/dotnetSdkTests.XXXXXXXX)")
 export DOTNET_CLI_HOME=$TestExecutionDirectory/.dotnet
 cp -a $HELIX_CORRELATION_PAYLOAD/t/TestExecutionDirectoryFiles/. $TestExecutionDirectory/
@@ -22,15 +28,17 @@ dotnet new --debug:ephemeral-hive
 
 dotnet nuget list source --configfile $TestExecutionDirectory/NuGet.config
 dotnet nuget add source $TestExecutionDirectory/Testpackages --configfile $TestExecutionDirectory/NuGet.config
-#Remove feeds not needed for tests
-dotnet nuget remove source dotnet6-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet6-internal-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet7-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet7-internal-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source richnav --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source vs-impl --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-libraries-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-tools-transport --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-libraries --configfile $TestExecutionDirectory/NuGet.config
-dotnet nuget remove source dotnet-eng --configfile $TestExecutionDirectory/NuGet.config
+# Remove feeds not needed for tests. Use || true to avoid errors when a source
+# doesn't exist (e.g. internal-transport feeds are only present in internal builds).
+dotnet nuget remove source dotnet6-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet6-internal-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet7-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet7-internal-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source richnav --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source vs-impl --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-libraries-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-tools-transport --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-libraries --configfile $TestExecutionDirectory/NuGet.config || true
+dotnet nuget remove source dotnet-eng --configfile $TestExecutionDirectory/NuGet.config || true
 dotnet nuget list source --configfile $TestExecutionDirectory/NuGet.config
+

src/BlazorWasmSdk/Tasks/GZipCompress.cs

@@ -20,6 +20,10 @@ public class GZipCompress : Task
         [Required]
         public string OutputDirectory { get; set; }
 
+        // Retry count for transient file I/O errors (e.g., antivirus locks on CI machines).
+        private const int MaxRetries = 3;
+        private const int RetryDelayMs = 200;
+
         public override bool Execute()
         {
             CompressedFiles = new ITaskItem[FilesToCompress.Length];
@@ -56,18 +60,31 @@ public override bool Execute()
                     Log.LogMessage(MessageImportance.Low, "Compressing '{0}' because file is newer than '{1}'.", inputFullPath, outputRelativePath);
                 }
 
-                try
+                // Retry on IOException to handle transient file locks from antivirus, file
+                // indexing, or parallel MSBuild nodes on CI machines (see dotnet/sdk#53424).
+                for (int attempt = 1; attempt <= MaxRetries; attempt++)
                 {
-                    using var sourceStream = File.OpenRead(file.ItemSpec);
-                    using var fileStream = File.Create(outputRelativePath);
-                    using var stream = new GZipStream(fileStream, CompressionLevel.Optimal);
+                    try
+                    {
+                        using var sourceStream = File.OpenRead(file.ItemSpec);
+                        using var fileStream = File.Create(outputRelativePath);
+                        using var stream = new GZipStream(fileStream, CompressionLevel.Optimal);
 
-                    sourceStream.CopyTo(stream);
-                }
-                catch (Exception e)
-                {
-                    Log.LogErrorFromException(e);
-                    return;
+                        sourceStream.CopyTo(stream);
+                        return; // Success
+                    }
+                    catch (IOException) when (attempt < MaxRetries)
+                    {
+                        Log.LogMessage(MessageImportance.Low,
+                            "Retrying compression of '{0}' (attempt {1}/{2}) due to transient I/O error.",
+                            file.ItemSpec, attempt, MaxRetries);
+                        Thread.Sleep(RetryDelayMs * attempt);
+                    }
+                    catch (Exception e)
+                    {
+                        Log.LogErrorFromException(e);
+                        return;
+                    }
                 }
             });
 

test/Microsoft.NET.Sdk.Razor.Tool.Tests/DefaultRequestDispatcherTest.cs

@@ -358,7 +358,7 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout
                         return connectionTask;
                     }
 
-                    readySource.SetResult(true);
+                    readySource.TrySetResult(true);
                     return new TaskCompletionSource<Connection>().Task;
                 });
 
@@ -382,11 +382,18 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout
                 }
             };
             var keepAlive = TimeSpan.FromSeconds(1);
-            var dispatcherTask = Task.Run(() =>
+
+            // Use Task.Factory.StartNew with LongRunning to run the dispatcher on a dedicated
+            // OS thread instead of a thread pool thread. The dispatcher's Run() method uses
+            // blocking Task.WaitAny() which permanently blocks its thread. On Helix CI agents
+            // running many tests in parallel, blocking a thread pool thread contributes to pool
+            // starvation, which prevents Task.Delay timer callbacks from firing, causing the
+            // keep-alive timeout to never complete and the test to hang indefinitely.
+            var dispatcherTask = Task.Factory.StartNew(() =>
             {
                 var dispatcher = new DefaultRequestDispatcher(connectionHost.Object, compilerHost, CancellationToken.None, eventBus, keepAlive);
                 dispatcher.Run();
-            });
+            }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default);
 
             // Wait for all connections to be created.
             await readySource.Task;
@@ -402,7 +409,10 @@ public async Task Dispatcher_ProcessSimultaneousConnections_HitsKeepAliveTimeout
 
             // Act
             // Now dispatcher should be in an idle state with no active connections.
-            await dispatcherTask;
+            // Use WaitAsync as a safety net: if the keep-alive timeout still can't fire
+            // (e.g. extreme thread pool starvation), fail the test after 60s instead of
+            // hanging for 60+ minutes and blocking the entire CI job.
+            await dispatcherTask.WaitAsync(TimeSpan.FromSeconds(60));
 
             // Assert
             Assert.False(eventBus.HasDetectedBadConnection);

test/TestAssets/Directory.Build.targets

@@ -1,4 +1,21 @@
 <!-- Prevent test asset projects from picking up the repo's root Directory.Build.targets. -->
 <Project>
 
+  <!-- For packable Exe projects (DotNetCliToolReference tools targeting netcoreapp2.2),
+       include the auto-generated runtimeconfig.json in the NuGet package so the dotnet
+       host can find it adjacent to the DLL. This enables RollForward=LatestMajor to
+       work correctly, allowing tools to run on machines that only have .NET 6.0+
+       installed (common on Helix CI agents that lack .NET Core 2.2). Without this,
+       tools fail with FrameworkMissingFailure (exit code 0x80008096) because the host
+       cannot roll forward from 2.2.0 without a runtimeconfig.json specifying the
+       rollForward policy. -->
+  <Target Name="IncludeRuntimeConfigInPackage"
+          AfterTargets="Build"
+          Condition="'$(OutputType)' == 'Exe' AND '$(IsPackable)' == 'true' AND '$(GenerateRuntimeConfigurationFiles)' != 'false'">
+    <ItemGroup>
+      <BuildOutputInPackage Include="$(ProjectRuntimeConfigFilePath)"
+                            Condition="'$(ProjectRuntimeConfigFilePath)' != '' AND Exists('$(ProjectRuntimeConfigFilePath)')" />
+    </ItemGroup>
+  </Target>
+
 </Project>