Skip to content

Simplify retry extension logic to use built-in filters #7511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Evangelink wants to merge 4 commits into
base: main
Choose a base branch
from
+156 −152
Open

Simplify retry extension logic to use built-in filters #7511

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7ca39a4
Simplify retry extension logic to use built-in filters
Evangelink Mar 9, 2026
ca8df8e
Address review comments
Evangelink Mar 9, 2026
a56ca61
Fix diagnostic
Evangelink Mar 9, 2026
9fe7078
Fixes
Evangelink Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 0 additions & 50 deletions src/Platform/Microsoft.Testing.Extensions.Retry/RetryExecutionFilterFactory.cs
View file
Open in desktop

This file was deleted.

10 changes: 0 additions & 10 deletions src/Platform/Microsoft.Testing.Extensions.Retry/RetryExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using Microsoft.Testing.Extensions.Policy.Resources;
using Microsoft.Testing.Platform.Builder;
using Microsoft.Testing.Platform.Extensions;
using Microsoft.Testing.Platform.TestHost;

namespace Microsoft.Testing.Extensions;

Expand Down Expand Up @@ -38,14 +37,5 @@ CompositeExtensionFactory<RetryDataConsumer> compositeExtensionFactory

builder.TestHostOrchestrator
.AddTestHostOrchestrator(serviceProvider => new RetryOrchestrator(serviceProvider));

if (builder.TestHost is not TestHostManager testHostManager)
{
throw new InvalidOperationException(
ExtensionResources.RetryProviderRequiresDefaultTestHostManagerErrorMessage);
}

testHostManager
.AddTestExecutionFilterFactory(serviceProvider => new RetryExecutionFilterFactory(serviceProvider));
}
}
37 changes: 37 additions & 0 deletions src/Platform/Microsoft.Testing.Extensions.Retry/RetryOrchestrator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,15 @@ public async Task<int> OrchestrateTestHostExecutionAsync(CancellationToken cance
finalArguments.Add($"--{RetryCommandLineOptionsProvider.RetryFailedTestsPipeNameOptionName}");
finalArguments.Add(retryFailedTestsPipeServer.PipeName);

// When retrying, replace any existing test filter with --filter-uid for the failed tests
if (lastListOfFailedId is { Length: > 0 })
{
RemoveOption(finalArguments, TreeNodeFilterCommandLineOptionsProvider.TreenodeFilter);
RemoveOption(finalArguments, PlatformCommandLineProvider.FilterUidOptionKey);
finalArguments.Add($"--{PlatformCommandLineProvider.FilterUidOptionKey}");
finalArguments.AddRange(lastListOfFailedId);
}

#if NET8_0_OR_GREATER
// On net8.0+, we can pass the arguments as a collection directly to ProcessStartInfo.
// When passing the collection, it's expected to be unescaped, so we pass what we have directly.
Expand Down Expand Up @@ -351,4 +360,32 @@ private static int GetOptionArgumentIndex(string optionName, string[] executable
index = Array.IndexOf(executableArgs, "--" + optionName);
return index >= 0 ? index : -1;
}

private static void RemoveOption(List<string> arguments, string optionName)
{
string longForm = $"--{optionName}";
string shortForm = $"-{optionName}";

// Remove all occurrences since options like --filter-uid can appear multiple times.
while (true)
{
int idx = arguments.IndexOf(longForm);
if (idx < 0)
{
idx = arguments.IndexOf(shortForm);
}

if (idx < 0)
{
break;
}

// Remove the option key and all its values
arguments.RemoveAt(idx);
while (idx < arguments.Count && !arguments[idx].StartsWith('-'))
{
arguments.RemoveAt(idx);
}
}
}
}
113 changes: 89 additions & 24 deletions ...tionTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/RetryFailedTestsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,56 @@ public async Task RetryFailedTests_PassingFromFirstTime_UsingTestTarget_MoveFile
}
}

[TestMethod]
[DynamicData(nameof(TargetFrameworks.NetForDynamicData), typeof(TargetFrameworks))]
public async Task RetryFailedTests_WithPreexistingFilterUid_ReplacesFilterOnRetry(string tfm)
{
var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm);
string resultDirectory = Path.Combine(testHost.DirectoryName, Guid.NewGuid().ToString("N"));

// Use --filter-uid to select tests 1 and 2. Test 1 will fail on first attempt, pass on second.
TestHostResult testHostResult = await testHost.ExecuteAsync(
$"--retry-failed-tests 3 --filter-uid 1 --filter-uid 2 --results-directory {resultDirectory}",
new()
{
{ EnvironmentVariableConstants.TESTINGPLATFORM_TELEMETRY_OPTOUT, "1" },
{ "METHOD1", "1" },
{ "RESULTDIR", resultDirectory },
},
cancellationToken: TestContext.CancellationToken);

testHostResult.AssertExitCodeIs(ExitCodes.Success);
testHostResult.AssertOutputContains("Tests suite completed successfully in 2 attempts");

// The retry attempt should only retry the failed test (UID 1), not all originally filtered tests.
testHostResult.AssertOutputContains("Tests suite failed, total failed tests: 1, exit code: 2, attempt: 1/4");
}

[TestMethod]
[DynamicData(nameof(TargetFrameworks.NetForDynamicData), typeof(TargetFrameworks))]
public async Task RetryFailedTests_WithPreexistingTreenodeFilter_ReplacesFilterOnRetry(string tfm)
{
var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm);
string resultDirectory = Path.Combine(testHost.DirectoryName, Guid.NewGuid().ToString("N"));

// Use --treenode-filter to select all tests. Test 1 will fail on first attempt, pass on second.
TestHostResult testHostResult = await testHost.ExecuteAsync(
$"--retry-failed-tests 3 --treenode-filter /** --results-directory {resultDirectory}",
new()
{
{ EnvironmentVariableConstants.TESTINGPLATFORM_TELEMETRY_OPTOUT, "1" },
{ "METHOD1", "1" },
{ "RESULTDIR", resultDirectory },
},
cancellationToken: TestContext.CancellationToken);

testHostResult.AssertExitCodeIs(ExitCodes.Success);
testHostResult.AssertOutputContains("Tests suite completed successfully in 2 attempts");

// The retry attempt should only retry the failed test (UID 1), not all tests matching the tree filter.
testHostResult.AssertOutputContains("Tests suite failed, total failed tests: 1, exit code: 2, attempt: 1/4");
}

public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder)
{
public string TargetAssetPath => GetAssetPath(AssetName);
Expand Down Expand Up @@ -252,6 +302,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.
using Microsoft.Testing.Platform.Extensions.Messages;
using Microsoft.Testing.Platform.Extensions.TestFramework;
using Microsoft.Testing.Platform.MSBuild;
using Microsoft.Testing.Platform.Requests;
using Microsoft.Testing.Platform.Services;

public class Program
Expand Down Expand Up @@ -306,46 +357,60 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
string resultDir = Environment.GetEnvironmentVariable("RESULTDIR")!;
bool crash = Environment.GetEnvironmentVariable("CRASH") == "1";

var uidFilter = (context.Request as TestExecutionRequest)?.Filter as TestNodeUidListFilter;
Copy link
Member

@Youssef1313 Youssef1313 Mar 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering how did this test work before? How was it respecting the retry filter?


var testMethod1Identifier = new TestMethodIdentifierProperty(string.Empty, string.Empty, "DummyClassName", "TestMethod1", 0, Array.Empty<string>(), string.Empty);
var testMethod2Identifier = new TestMethodIdentifierProperty(string.Empty, string.Empty, "DummyClassName", "TestMethod2", 0, Array.Empty<string>(), string.Empty);
var testMethod3Identifier = new TestMethodIdentifierProperty(string.Empty, string.Empty, "DummyClassName", "TestMethod3", 0, Array.Empty<string>(), string.Empty);

if (TestMethod1(fail, resultDir, crash))
if (IsIncluded(uidFilter, "1"))
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "1", DisplayName = "TestMethod1", Properties = new(PassedTestNodeStateProperty.CachedInstance, testMethod1Identifier) }));
}
else
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "1", DisplayName = "TestMethod1", Properties = new(new FailedTestNodeStateProperty(), testMethod1Identifier) }));
if (TestMethod1(fail, resultDir, crash))
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "1", DisplayName = "TestMethod1", Properties = new(PassedTestNodeStateProperty.CachedInstance, testMethod1Identifier) }));
}
else
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "1", DisplayName = "TestMethod1", Properties = new(new FailedTestNodeStateProperty(), testMethod1Identifier) }));
}
}

if (TestMethod2(fail, resultDir))
if (IsIncluded(uidFilter, "2"))
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "2", DisplayName = "TestMethod2", Properties = new(PassedTestNodeStateProperty.CachedInstance, testMethod2Identifier) }));
}
else
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "2", DisplayName = "TestMethod2", Properties = new(new FailedTestNodeStateProperty(), testMethod2Identifier) }));
if (TestMethod2(fail, resultDir))
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "2", DisplayName = "TestMethod2", Properties = new(PassedTestNodeStateProperty.CachedInstance, testMethod2Identifier) }));
}
else
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "2", DisplayName = "TestMethod2", Properties = new(new FailedTestNodeStateProperty(), testMethod2Identifier) }));
}
}

if (TestMethod3(fail, resultDir))
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "3", DisplayName = "TestMethod3", Properties = new(PassedTestNodeStateProperty.CachedInstance, testMethod3Identifier) }));
}
else
if (IsIncluded(uidFilter, "3"))
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "3", DisplayName = "TestMethod3", Properties = new(new FailedTestNodeStateProperty(), testMethod3Identifier) }));
if (TestMethod3(fail, resultDir))
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "3", DisplayName = "TestMethod3", Properties = new(PassedTestNodeStateProperty.CachedInstance, testMethod3Identifier) }));
}
else
{
await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid,
new TestNode() { Uid = "3", DisplayName = "TestMethod3", Properties = new(new FailedTestNodeStateProperty(), testMethod3Identifier) }));
}
}

context.Complete();
}

private static bool IsIncluded(TestNodeUidListFilter? filter, string uid)
=> filter is null || filter.TestNodeUids.Any(n => n.Value == uid);

private bool TestMethod1(bool fail, string resultDir, bool crash)
{
if (crash)
Expand Down
Loading