diff --git a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj index b6d063c7d2..d703a6f40c 100644 --- a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj +++ b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj @@ -51,6 +51,11 @@ $(DefineConstants);WINDOWS_UWP + + + $(DefineConstants);WIN_UI + + diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs index 55e200cd73..2cb24b944d 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs @@ -9,8 +9,10 @@ using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.Telemetry; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Helpers; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -41,7 +43,7 @@ protected override Task SynchronizedDiscoverTestsAsync(VSTestDiscoverTestExecuti Debugger.Launch(); } - new MSTestDiscoverer().DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration); + new MSTestDiscoverer(new TestSourceHandler(), CreateTelemetrySender()).DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration); return Task.CompletedTask; } @@ -55,7 +57,7 @@ protected override async Task SynchronizedRunTestsAsync(VSTestRunTestExecutionRe Debugger.Launch(); } - MSTestExecutor testExecutor = new(cancellationToken); + MSTestExecutor testExecutor = new(cancellationToken, CreateTelemetrySender()); await testExecutor.RunTestsAsync(request.AssemblyPaths, request.RunContext, request.FrameworkHandle, _configuration).ConfigureAwait(false); } @@ -103,5 +105,19 @@ private static TestMethodIdentifierProperty GetMethodIdentifierPropertyFromManag // Or alternatively, does VSTest object model expose the assembly full name somewhere? return new TestMethodIdentifierProperty(assemblyFullName: string.Empty, @namespace, typeName, methodName, arity, parameterTypes, returnTypeFullName: string.Empty); } + + [SuppressMessage("ApiDesign", "RS0030:Do not use banned APIs", Justification = "We can use MTP from this folder")] + private Func, Task>? CreateTelemetrySender() + { + ITelemetryInformation telemetryInformation = ServiceProvider.GetTelemetryInformation(); + if (!telemetryInformation.IsEnabled) + { + return null; + } + + ITelemetryCollector telemetryCollector = ServiceProvider.GetTelemetryCollector(); + + return (eventName, metrics) => telemetryCollector.LogEventAsync(eventName, metrics, CancellationToken.None); + } } #endif diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index 3a3f24bf76..a9b0500526 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -20,14 +20,24 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestDiscoverer : ITestDiscoverer { private readonly ITestSourceHandler _testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI + private readonly Func, Task>? _telemetrySender; +#endif public MSTestDiscoverer() : this(new TestSourceHandler()) { } - internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler) - => _testSourceHandler = testSourceHandler; + internal MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func, Task>? telemetrySender = null) + { + _testSourceHandler = testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI + _telemetrySender = telemetrySender; +#else + _ = telemetrySender; +#endif + } /// /// Discovers the tests available from the provided source. Not supported for .xap source. @@ -47,9 +57,26 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco Ensure.NotNull(logger); Ensure.NotNull(discoverySink); - if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + // Initialize telemetry collection if not already set (e.g. first call in the session) +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + _ = MSTestTelemetryDataCollector.EnsureInitialized(); + } +#endif + + try + { + if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + { + new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext); + } + } + finally { - new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext); +#if !WINDOWS_UWP && !WIN_UI + MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender).GetAwaiter().GetResult(); +#endif } } } diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs index 8fcb749311..fc20eba8d4 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs @@ -20,6 +20,9 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestExecutor : ITestExecutor { private readonly CancellationToken _cancellationToken; +#if !WINDOWS_UWP && !WIN_UI + private readonly Func, Task>? _telemetrySender; +#endif /// /// Token for canceling the test run. @@ -35,10 +38,15 @@ public MSTestExecutor() _cancellationToken = CancellationToken.None; } - internal MSTestExecutor(CancellationToken cancellationToken) + internal MSTestExecutor(CancellationToken cancellationToken, Func, Task>? telemetrySender = null) { TestExecutionManager = new TestExecutionManager(); _cancellationToken = cancellationToken; +#if !WINDOWS_UWP && !WIN_UI + _telemetrySender = telemetrySender; +#else + _ = telemetrySender; +#endif } /// @@ -105,12 +113,27 @@ internal async Task RunTestsAsync(IEnumerable? tests, IRunContext? run Ensure.NotNull(frameworkHandle); Ensure.NotNullOrEmpty(tests); + // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + _ = MSTestTelemetryDataCollector.EnsureInitialized(); + } +#endif + if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) { return; } - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + try + { + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? runContext, IFrameworkHandle? frameworkHandle, IConfiguration? configuration) @@ -123,6 +146,14 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run Ensure.NotNull(frameworkHandle); Ensure.NotNullOrEmpty(sources); + // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + _ = MSTestTelemetryDataCollector.EnsureInitialized(); + } +#endif + TestSourceHandler testSourceHandler = new(); if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) { @@ -130,7 +161,14 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run } sources = testSourceHandler.GetTestSources(sources); - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + try + { + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } /// @@ -139,6 +177,14 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run public void Cancel() => _testRunCancellationToken?.Cancel(); +#if !WINDOWS_UWP && !WIN_UI + private Task SendTelemetryAsync() + => MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender); +#else + private static Task SendTelemetryAsync() + => Task.CompletedTask; +#endif + private async Task RunTestsFromRightContextAsync(IFrameworkHandle frameworkHandle, Func runTestsAction) { ApartmentState? requestedApartmentState = MSTestSettings.RunConfigurationSettings.ExecutionApartmentState; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs index b3ea7a2df6..8e610696d4 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs @@ -145,7 +145,11 @@ internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFile var typeValidator = new TypeValidator(ReflectHelper, discoverInternals); var testMethodValidator = new TestMethodValidator(ReflectHelper, discoverInternals); +#if !WINDOWS_UWP && !WIN_UI + return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator, Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector.Current); +#else return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator); +#endif } private List DiscoverTestsInType( diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs index b151d6d594..dd8acb3eb8 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs @@ -19,7 +19,22 @@ internal class TypeEnumerator private readonly TypeValidator _typeValidator; private readonly TestMethodValidator _testMethodValidator; private readonly ReflectHelper _reflectHelper; +#if !WINDOWS_UWP && !WIN_UI + private readonly Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector? _telemetryDataCollector; +#endif +#if !WINDOWS_UWP && !WIN_UI + /// + /// Initializes a new instance of the class. + /// + /// The reflected type. + /// The name of the assembly being reflected. + /// An instance to reflection helper for type information. + /// The validator for test classes. + /// The validator for test methods. + /// Optional telemetry data collector for tracking API usage. + internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector? telemetryDataCollector = null) +#else /// /// Initializes a new instance of the class. /// @@ -29,12 +44,16 @@ internal class TypeEnumerator /// The validator for test classes. /// The validator for test methods. internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator) +#endif { _type = type; _assemblyFilePath = assemblyFilePath; _reflectHelper = reflectHelper; _typeValidator = typeValidator; _testMethodValidator = testMethodValidator; +#if !WINDOWS_UWP && !WIN_UI + _telemetryDataCollector = telemetryDataCollector; +#endif } /// @@ -49,6 +68,15 @@ internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflec return null; } + // Track class-level attributes for telemetry +#if !WINDOWS_UWP && !WIN_UI + if (_telemetryDataCollector is not null) + { + Attribute[] classAttributes = _reflectHelper.GetCustomAttributesCached(_type); + _telemetryDataCollector.TrackDiscoveredClass(classAttributes); + } +#endif + // If test class is valid, then get the tests return GetTests(warnings); } @@ -143,6 +171,9 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool classDisables }; Attribute[] attributes = _reflectHelper.GetCustomAttributesCached(method); +#if !WINDOWS_UWP && !WIN_UI + _telemetryDataCollector?.TrackDiscoveredMethod(attributes); +#endif TestMethodAttribute? testMethodAttribute = null; // Backward looping for backcompat. This used to be calls to _reflectHelper.GetFirstAttributeOrDefault diff --git a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs index 501d5559cd..d1d642a70f 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs @@ -295,6 +295,18 @@ internal static void PopulateSettings(IDiscoveryContext? context, IMessageLogger CurrentSettings = settings; RunConfigurationSettings = runConfigurationSettings; + + // Track configuration source for telemetry +#if !WINDOWS_UWP && !WIN_UI + if (MSTestTelemetryDataCollector.Current is { } telemetry) + { + telemetry.ConfigurationSource = configuration?["mstest"] is not null + ? "testconfig.json" + : !StringEx.IsNullOrEmpty(context?.RunSettings?.SettingsXml) + ? "runsettings" + : "none"; + } +#endif } /// diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs new file mode 100644 index 0000000000..8bab8353ff --- /dev/null +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -0,0 +1,375 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if !WINDOWS_UWP && !WIN_UI +using System.Security.Cryptography; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; + +/// +/// Collects and aggregates telemetry data about MSTest usage within a test session. +/// Captures settings, attribute usage, custom/inherited types, and assertion API usage. +/// +internal sealed class MSTestTelemetryDataCollector +{ + private readonly Dictionary _attributeCounts = []; + private readonly HashSet _customTestMethodTypes = []; + private readonly HashSet _customTestClassTypes = []; + +#pragma warning disable IDE0032 // Use auto property - Volatile.Read/Write requires a ref to a field + private static MSTestTelemetryDataCollector? s_current; +#pragma warning restore IDE0032 // Use auto property + + /// + /// Gets or sets the current telemetry data collector for the session. + /// Set at session start, cleared at session close. + /// + internal static MSTestTelemetryDataCollector? Current + { + get => Volatile.Read(ref s_current); + set => Volatile.Write(ref s_current, value); + } + + internal static MSTestTelemetryDataCollector EnsureInitialized() + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is not null) + { + return collector; + } + + collector = new MSTestTelemetryDataCollector(); + MSTestTelemetryDataCollector? existingCollector = Interlocked.CompareExchange(ref s_current, collector, null); + + return existingCollector ?? collector; + } + + /// + /// Gets a value indicating whether any data has been collected. + /// + internal bool HasData { get; private set; } + + /// + /// Checks whether telemetry collection is opted out via environment variables. + /// Mirrors the same checks as Microsoft.Testing.Platform's TelemetryManager. + /// + /// true if telemetry is opted out; false otherwise. + internal static bool IsTelemetryOptedOut() + { + string? telemetryOptOut = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); + if (telemetryOptOut is "1" or "true") + { + return true; + } + + string? cliTelemetryOptOut = Environment.GetEnvironmentVariable("DOTNET_CLI_TELEMETRY_OPTOUT"); + + return cliTelemetryOptOut is "1" or "true"; + } + + /// + /// Gets or sets the configuration source used for this session. + /// + internal string? ConfigurationSource { get; set; } + + /// + /// Records the attributes found on a test method during discovery. + /// + /// The cached attributes from the method. + internal void TrackDiscoveredMethod(Attribute[] attributes) + { + HasData = true; + + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + string attributeName = attributeType.Name; + + // Track custom/inherited TestMethodAttribute types (store anonymized hash) + if (attribute is TestMethodAttribute && attributeType != typeof(TestMethodAttribute)) + { + _customTestMethodTypes.Add(AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + _customTestClassTypes.Add(AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track attribute usage counts by base type name + string trackingName = attribute switch + { + TestMethodAttribute => nameof(TestMethodAttribute), + TestClassAttribute => nameof(TestClassAttribute), + DataRowAttribute => nameof(DataRowAttribute), + DynamicDataAttribute => nameof(DynamicDataAttribute), + TimeoutAttribute => nameof(TimeoutAttribute), + IgnoreAttribute => nameof(IgnoreAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + RetryBaseAttribute => nameof(RetryBaseAttribute), + ConditionBaseAttribute => nameof(ConditionBaseAttribute), + TestCategoryAttribute => nameof(TestCategoryAttribute), +#if !WIN_UI + DeploymentItemAttribute => nameof(DeploymentItemAttribute), +#endif + _ => attributeName, + }; + + _attributeCounts[trackingName] = _attributeCounts.TryGetValue(trackingName, out long count) + ? count + 1 + : 1; + } + } + + /// + /// Records the attributes found on a test class during discovery. + /// + /// The cached attributes from the class. + internal void TrackDiscoveredClass(Attribute[] attributes) + { + HasData = true; + + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + _customTestClassTypes.Add(AnonymizeString(attributeType.FullName ?? attributeType.Name)); + } + + string? trackingName = attribute switch + { + TestClassAttribute => nameof(TestClassAttribute), + ParallelizeAttribute => nameof(ParallelizeAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + _ => null, + }; + + if (trackingName is not null) + { + _attributeCounts[trackingName] = _attributeCounts.TryGetValue(trackingName, out long count) + ? count + 1 + : 1; + } + } + } + + /// + /// Builds the telemetry metrics dictionary for sending via the telemetry collector. + /// + /// A dictionary of telemetry key-value pairs. + internal Dictionary BuildMetrics() + { + Dictionary metrics = []; + + // Settings + AddSettingsMetrics(metrics); + + // Configuration source (runsettings, testconfig.json, or none) + if (ConfigurationSource is not null) + { + metrics["mstest.config_source"] = ConfigurationSource; + } + + // Attribute usage (aggregated counts as JSON) + if (_attributeCounts.Count > 0) + { + metrics["mstest.attribute_usage"] = SerializeDictionary(_attributeCounts); + } + + // Custom/inherited types (anonymized names) + if (_customTestMethodTypes.Count > 0) + { + metrics["mstest.custom_test_method_types"] = SerializeCollection(_customTestMethodTypes); + } + + if (_customTestClassTypes.Count > 0) + { + metrics["mstest.custom_test_class_types"] = SerializeCollection(_customTestClassTypes); + } + + // Assertion usage (drain the static counters) + Dictionary assertionCounts = TelemetryCollector.DrainAssertionCallCounts(); + if (assertionCounts.Count > 0) + { + metrics["mstest.assertion_usage"] = SerializeDictionary(assertionCounts); + } + + return metrics; + } + + private static string SerializeCollection(IEnumerable values) + { + System.Text.StringBuilder builder = new("["); + bool isFirst = true; + + foreach (string value in values) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value); + isFirst = false; + } + + builder.Append(']'); + return builder.ToString(); + } + + private static string SerializeDictionary(Dictionary values) + { + System.Text.StringBuilder builder = new("{"); + bool isFirst = true; + + foreach (KeyValuePair value in values) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value.Key); + builder.Append(':'); + builder.Append(value.Value.ToString(System.Globalization.CultureInfo.InvariantCulture)); + isFirst = false; + } + + builder.Append('}'); + return builder.ToString(); + } + + private static void AppendJsonString(System.Text.StringBuilder builder, string value) + { + builder.Append('"'); + + foreach (char character in value) + { + switch (character) + { + case '"': + builder.Append("\\\""); + break; + case '\\': + builder.Append("\\\\"); + break; + case '\b': + builder.Append("\\b"); + break; + case '\f': + builder.Append("\\f"); + break; + case '\n': + builder.Append("\\n"); + break; + case '\r': + builder.Append("\\r"); + break; + case '\t': + builder.Append("\\t"); + break; + default: + if (char.IsControl(character)) + { + builder.Append("\\u"); + builder.Append(((int)character).ToString("x4", System.Globalization.CultureInfo.InvariantCulture)); + } + else + { + builder.Append(character); + } + + break; + } + } + + builder.Append('"'); + } + + private static void AddSettingsMetrics(Dictionary metrics) + { + MSTestSettings settings = MSTestSettings.CurrentSettings; + + // Parallelization + metrics["mstest.setting.parallelization_enabled"] = !settings.DisableParallelization; + if (settings.ParallelizationScope is not null) + { + metrics["mstest.setting.parallelization_scope"] = settings.ParallelizationScope.Value.ToString(); + } + + if (settings.ParallelizationWorkers is not null) + { + metrics["mstest.setting.parallelization_workers"] = settings.ParallelizationWorkers.Value; + } + + // Timeouts + metrics["mstest.setting.test_timeout"] = settings.TestTimeout; + metrics["mstest.setting.assembly_initialize_timeout"] = settings.AssemblyInitializeTimeout; + metrics["mstest.setting.assembly_cleanup_timeout"] = settings.AssemblyCleanupTimeout; + metrics["mstest.setting.class_initialize_timeout"] = settings.ClassInitializeTimeout; + metrics["mstest.setting.class_cleanup_timeout"] = settings.ClassCleanupTimeout; + metrics["mstest.setting.test_initialize_timeout"] = settings.TestInitializeTimeout; + metrics["mstest.setting.test_cleanup_timeout"] = settings.TestCleanupTimeout; + metrics["mstest.setting.cooperative_cancellation"] = settings.CooperativeCancellationTimeout; + + // Behavior + metrics["mstest.setting.map_inconclusive_to_failed"] = settings.MapInconclusiveToFailed; + metrics["mstest.setting.map_not_runnable_to_failed"] = settings.MapNotRunnableToFailed; + metrics["mstest.setting.treat_discovery_warnings_as_errors"] = settings.TreatDiscoveryWarningsAsErrors; + metrics["mstest.setting.consider_empty_data_source_as_inconclusive"] = settings.ConsiderEmptyDataSourceAsInconclusive; + metrics["mstest.setting.order_tests_by_name"] = settings.OrderTestsByNameInClass; + metrics["mstest.setting.capture_debug_traces"] = settings.CaptureDebugTraces; + metrics["mstest.setting.has_test_settings_file"] = settings.TestSettingsFile is not null; + } + + private static string AnonymizeString(string value) + { +#if NET + byte[] hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(hash); +#else + using var sha256 = SHA256.Create(); + byte[] hash = sha256.ComputeHash(System.Text.Encoding.UTF8.GetBytes(value)); + return BitConverter.ToString(hash).Replace("-", string.Empty); +#endif + } + + /// + /// Sends collected telemetry via the provided sender delegate and resets the current collector. + /// Safe to call even when no sender is available (no-op). + /// + /// Optional delegate to send telemetry. If null, telemetry is silently discarded. + internal static async Task SendTelemetryAndResetAsync(Func, Task>? telemetrySender) + { + try + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is not { HasData: true } || telemetrySender is null) + { + TelemetryCollector.DrainAssertionCallCounts(); + return; + } + + Dictionary metrics = collector.BuildMetrics(); + if (metrics.Count > 0) + { + await telemetrySender("dotnet/testingplatform/mstest/sessionexit", metrics).ConfigureAwait(false); + } + } + catch (Exception) + { + // Telemetry should never cause test failures + } + finally + { + Current = null; + } + } +} +#endif diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 0c69386a00..869e113dcf 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -59,6 +59,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs index 5cd7fe9939..2938a04074 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs @@ -48,6 +48,7 @@ internal void ComputeAssertion(string expectedExpression, string actualExpressio { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "expected", expectedExpression, "actual", actualExpression) + " "); ThrowAssertAreEqualFailed(_expected, _actual, _builder.ToString()); } @@ -115,6 +116,7 @@ internal void ComputeAssertion(string notExpectedExpression, string actualExpres { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "notExpected", notExpectedExpression, "actual", actualExpression) + " "); ThrowAssertAreNotEqualFailed(_notExpected, _actual, _builder.ToString()); } @@ -221,6 +223,7 @@ internal void ComputeAssertion(string expectedExpression, string actualExpressio { if (_failAction is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); _builder!.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "expected", expectedExpression, "actual", actualExpression) + " "); _failAction.Invoke(_builder!.ToString()); } @@ -327,6 +330,7 @@ internal void ComputeAssertion(string notExpectedExpression, string actualExpres { if (_failAction is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); _builder!.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "notExpected", notExpectedExpression, "actual", actualExpression) + " "); _failAction.Invoke(_builder!.ToString()); } @@ -483,6 +487,8 @@ public static void AreEqual(T? expected, T? actual, IEqualityComparer? com /// public static void AreEqual(T? expected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (!AreEqualFailing(expected, actual, comparer)) { return; @@ -786,6 +792,8 @@ public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (!AreNotEqualFailing(notExpected, actual, comparer)) { return; @@ -835,6 +843,8 @@ public static void AreEqual(float expected, float actual, float delta, [Interpol /// public static void AreEqual(float expected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -882,6 +892,8 @@ public static void AreNotEqual(float notExpected, float actual, float delta, [In /// public static void AreNotEqual(float notExpected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -950,6 +962,8 @@ public static void AreEqual(decimal expected, decimal actual, decimal delta, [In /// public static void AreEqual(decimal expected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -997,6 +1011,8 @@ public static void AreNotEqual(decimal notExpected, decimal actual, decimal delt /// public static void AreNotEqual(decimal notExpected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -1047,6 +1063,8 @@ public static void AreEqual(long expected, long actual, long delta, [Interpolate /// public static void AreEqual(long expected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -1094,6 +1112,8 @@ public static void AreNotEqual(long notExpected, long actual, long delta, [Inter /// public static void AreNotEqual(long notExpected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -1143,6 +1163,8 @@ public static void AreEqual(double expected, double actual, double delta, [Inter /// public static void AreEqual(double expected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -1190,6 +1212,8 @@ public static void AreNotEqual(double notExpected, double actual, double delta, /// public static void AreNotEqual(double notExpected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -1316,6 +1340,8 @@ public static void AreEqual(string? expected, string? actual, bool ignoreCase, /// public static void AreEqual(string? expected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + CheckParameterNotNull(culture, "Assert.AreEqual", "culture"); if (!AreEqualFailing(expected, actual, ignoreCase, culture)) { @@ -1412,6 +1438,8 @@ public static void AreNotEqual(string? notExpected, string? actual, bool ignoreC /// public static void AreNotEqual(string? notExpected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + CheckParameterNotNull(culture, "Assert.AreNotEqual", "culture"); if (!AreNotEqualFailing(notExpected, actual, ignoreCase, culture)) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs index 93be91c7fd..90491fec73 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs @@ -38,6 +38,7 @@ internal void ComputeAssertion(string expectedExpression, string actualExpressio { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreSame"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "expected", expectedExpression, "actual", actualExpression) + " "); ThrowAssertAreSameFailed(_expected, _actual, _builder.ToString()); } @@ -94,6 +95,7 @@ internal void ComputeAssertion(string notExpectedExpression, string actualExpres { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreNotSame"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "notExpected", notExpectedExpression, "actual", actualExpression) + " "); ThrowAssertAreNotSameFailed(_builder.ToString()); } @@ -172,6 +174,8 @@ public static void AreSame(T? expected, T? actual, [InterpolatedStringHandler /// public static void AreSame(T? expected, T? actual, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreSame"); + if (!IsAreSameFailing(expected, actual)) { return; @@ -238,6 +242,8 @@ public static void AreNotSame(T? notExpected, T? actual, [InterpolatedStringH /// public static void AreNotSame(T? notExpected, T? actual, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotSame"); + if (IsAreNotSameFailing(notExpected, actual)) { ThrowAssertAreNotSameFailed(BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs index 19fb68e0a6..01b90d7b93 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs @@ -154,6 +154,8 @@ public static T ContainsSingle(IEnumerable collection, string? message = " /// The item that matches the predicate. public static T ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + T firstMatch = default!; int matchCount = 0; @@ -209,6 +211,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// The item that matches the predicate. public static object? ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + object? firstMatch = null; int matchCount = 0; @@ -273,6 +277,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// public static void Contains(T expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -296,6 +302,8 @@ public static void Contains(T expected, IEnumerable collection, string? me /// public static void Contains(object? expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); foreach (object? item in collection) @@ -328,6 +336,8 @@ public static void Contains(object? expected, IEnumerable collection, string? me /// public static void Contains(T expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected, comparer)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -352,6 +362,8 @@ public static void Contains(T expected, IEnumerable collection, IEqualityC /// public static void Contains(object? expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(comparer, "Assert.Contains", "comparer"); @@ -384,6 +396,8 @@ public static void Contains(object? expected, IEnumerable collection, IEqualityC /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -407,6 +421,8 @@ public static void Contains(Func predicate, IEnumerable collectio /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(predicate, "Assert.Contains", "predicate"); @@ -486,6 +502,8 @@ public static void Contains(string substring, string value, string? message = "" /// public static void Contains(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(value, "Assert.Contains", "value"); CheckParameterNotNull(substring, "Assert.Contains", "substring"); @@ -518,6 +536,8 @@ public static void Contains(string substring, string value, StringComparison com /// public static void DoesNotContain(T notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -541,6 +561,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, s /// public static void DoesNotContain(object? notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); foreach (object? item in collection) @@ -571,6 +593,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, s /// public static void DoesNotContain(T notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected, comparer)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -595,6 +619,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, I /// public static void DoesNotContain(object? notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(comparer, "Assert.DoesNotContain", "comparer"); @@ -625,6 +651,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, I /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -648,6 +676,8 @@ public static void DoesNotContain(Func predicate, IEnumerable col /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(predicate, "Assert.DoesNotContain", "predicate"); @@ -725,6 +755,8 @@ public static void DoesNotContain(string substring, string value, string? messag /// public static void DoesNotContain(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(value, "Assert.DoesNotContain", "value"); CheckParameterNotNull(substring, "Assert.DoesNotContain", "substring"); @@ -764,6 +796,8 @@ public static void DoesNotContain(string substring, string value, StringComparis public static void IsInRange(T minValue, T maxValue, T value, string? message = "", [CallerArgumentExpression(nameof(minValue))] string minValueExpression = "", [CallerArgumentExpression(nameof(maxValue))] string maxValueExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsInRange"); + if (maxValue.CompareTo(minValue) < 0) { throw new ArgumentOutOfRangeException(nameof(maxValue), FrameworkMessages.IsInRangeMaxValueMustBeGreaterThanOrEqualMinValue); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs index 555d341686..0a9200b2c2 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs @@ -47,6 +47,7 @@ internal void ComputeAssertion(string assertionName, string collectionExpression { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertionName)); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "collection", collectionExpression) + " "); ThrowAssertCountFailed(assertionName, _expectedCount, _actualCount, _builder.ToString()); } @@ -115,6 +116,7 @@ internal void ComputeAssertion(string collectionExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "collection", collectionExpression) + " "); ThrowAssertIsNotEmptyFailed(_builder.ToString()); } @@ -197,6 +199,8 @@ public static void IsNotEmpty(IEnumerable collection, [InterpolatedStringH /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Any()) { return; @@ -217,6 +221,8 @@ public static void IsNotEmpty(IEnumerable collection, string? message = "" /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Cast().Any()) { return; @@ -320,6 +326,8 @@ public static void IsEmpty(IEnumerable collection, string? message = "", [Caller private static void HasCount(string assertionName, int expected, IEnumerable collection, string? message, string collectionExpression) { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertionName)); + int actualCount = collection.Count(); if (actualCount == expected) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs index 772c1d1e48..48485bc71f 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs @@ -72,6 +72,8 @@ public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? /// public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedSuffix))] string expectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.EndsWith"); + CheckParameterNotNull(value, "Assert.EndsWith", "value"); CheckParameterNotNull(expectedSuffix, "Assert.EndsWith", "expectedSuffix"); if (!value.EndsWith(expectedSuffix, comparisonType)) @@ -146,6 +148,8 @@ public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] /// public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedSuffix))] string notExpectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotEndWith"); + CheckParameterNotNull(value, "Assert.DoesNotEndWith", "value"); CheckParameterNotNull(notExpectedSuffix, "Assert.DoesNotEndWith", "notExpectedSuffix"); if (value.EndsWith(notExpectedSuffix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs index 3a3a4ebd24..826a93610b 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs @@ -22,5 +22,8 @@ public sealed partial class Assert /// [DoesNotReturn] public static void Fail(string message = "") - => ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + { + TelemetryCollector.TrackAssertionCall("Assert.Fail"); + ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + } } diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs index a953f5cd33..fa75a25c2e 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs @@ -44,6 +44,8 @@ public sealed partial class Assert public static void IsGreaterThan(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThan"); + if (value.CompareTo(lowerBound) > 0) { return; @@ -89,6 +91,8 @@ public static void IsGreaterThan(T lowerBound, T value, string? message = "", public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThanOrEqualTo"); + if (value.CompareTo(lowerBound) >= 0) { return; @@ -134,6 +138,8 @@ public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? mess public static void IsLessThan(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThan"); + if (value.CompareTo(upperBound) < 0) { return; @@ -179,6 +185,8 @@ public static void IsLessThan(T upperBound, T value, string? message = "", [C public static void IsLessThanOrEqualTo(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThanOrEqualTo"); + if (value.CompareTo(upperBound) <= 0) { return; @@ -216,6 +224,8 @@ public static void IsLessThanOrEqualTo(T upperBound, T value, string? message public static void IsPositive(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsPositive"); + var zero = default(T); // Handle special case for floating point NaN values @@ -270,6 +280,8 @@ public static void IsPositive(T value, string? message = "", [CallerArgumentE public static void IsNegative(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsNegative"); + var zero = default(T); // Handle special case for floating point NaN values diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs index cca665dbb8..8948ecb71c 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs @@ -23,6 +23,8 @@ public sealed partial class Assert [DoesNotReturn] public static void Inconclusive(string message = "") { + TelemetryCollector.TrackAssertionCall("Assert.Inconclusive"); + string userMessage = BuildUserMessage(message); throw new AssertInconclusiveException( string.Format(CultureInfo.CurrentCulture, FrameworkMessages.AssertionFailed, "Assert.Inconclusive", userMessage)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs index 144f17cdbe..d23638cfa8 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs @@ -38,6 +38,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsExactInstanceOfTypeFailed(_value, _expectedType, _builder.ToString()); } @@ -98,6 +99,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsExactInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -160,6 +162,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsNotExactInstanceOfTypeFailed(_value, _wrongType, _builder.ToString()); } @@ -220,6 +223,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsNotExactInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -289,6 +293,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsExactInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); + if (IsExactInstanceOfTypeFailing(value, expectedType)) { ThrowAssertIsExactInstanceOfTypeFailed(value, expectedType, BuildUserMessageForValueExpression(message, valueExpression)); @@ -371,6 +377,8 @@ private static void ThrowAssertIsExactInstanceOfTypeFailed(object? value, Type? /// public static void IsNotExactInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); + if (IsNotExactInstanceOfTypeFailing(value, wrongType)) { ThrowAssertIsNotExactInstanceOfTypeFailed(value, wrongType, BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs index bdd68f1607..c181d5c619 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs @@ -38,6 +38,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsInstanceOfTypeFailed(_value, _expectedType, _builder.ToString()); } @@ -98,6 +99,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -160,6 +162,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsNotInstanceOfTypeFailed(_value, _wrongType, _builder.ToString()); } @@ -220,6 +223,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsNotInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -290,6 +294,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); + if (IsInstanceOfTypeFailing(value, expectedType)) { ThrowAssertIsInstanceOfTypeFailed(value, expectedType, BuildUserMessageForValueExpression(message, valueExpression)); @@ -374,6 +380,8 @@ private static void ThrowAssertIsInstanceOfTypeFailed(object? value, Type? expec /// public static void IsNotInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); + if (IsNotInstanceOfTypeFailing(value, wrongType)) { ThrowAssertIsNotInstanceOfTypeFailed(value, wrongType, BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs index fd6db5b3d4..6474065b49 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs @@ -36,6 +36,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNull"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsNullFailed(_builder.ToString()); } @@ -90,6 +91,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotNull"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ThrowAssertIsNotNullFailed(_builder.ToString()); } @@ -150,6 +152,8 @@ public static void IsNull(object? value, [InterpolatedStringHandlerArgument(name /// public static void IsNull(object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNull"); + if (IsNullFailing(value)) { ThrowAssertIsNullFailed(BuildUserMessageForValueExpression(message, valueExpression)); @@ -189,6 +193,8 @@ public static void IsNotNull([NotNull] object? value, [InterpolatedStringHandler /// public static void IsNotNull([NotNull] object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotNull"); + if (IsNotNullFailing(value)) { ThrowAssertIsNotNullFailed(BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs index 1c31f3add1..fed8cffa11 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs @@ -36,6 +36,7 @@ internal void ComputeAssertion(string conditionExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsTrue"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "condition", conditionExpression) + " "); ThrowAssertIsTrueFailed(_builder.ToString()); } @@ -88,6 +89,7 @@ internal void ComputeAssertion(string conditionExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsFalse"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "condition", conditionExpression) + " "); ThrowAssertIsFalseFailed(_builder.ToString()); } @@ -148,6 +150,8 @@ public static void IsTrue([DoesNotReturnIf(false)] bool? condition, [Interpolate /// public static void IsTrue([DoesNotReturnIf(false)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsTrue"); + if (IsTrueFailing(condition)) { ThrowAssertIsTrueFailed(BuildUserMessageForConditionExpression(message, conditionExpression)); @@ -186,6 +190,8 @@ public static void IsFalse([DoesNotReturnIf(true)] bool? condition, [Interpolate /// public static void IsFalse([DoesNotReturnIf(true)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsFalse"); + if (IsFalseFailing(condition)) { ThrowAssertIsFalseFailed(BuildUserMessageForConditionExpression(message, conditionExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs index b204970b6d..28e5ba1ee0 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs @@ -39,6 +39,8 @@ public sealed partial class Assert /// public static void MatchesRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.MatchesRegex"); + CheckParameterNotNull(value, "Assert.MatchesRegex", "value"); CheckParameterNotNull(pattern, "Assert.MatchesRegex", "pattern"); @@ -115,6 +117,8 @@ public static void MatchesRegex([NotNull] string? pattern, [NotNull] string? val /// public static void DoesNotMatchRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotMatchRegex"); + CheckParameterNotNull(value, "Assert.DoesNotMatchRegex", "value"); CheckParameterNotNull(pattern, "Assert.DoesNotMatchRegex", "pattern"); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs index 6f5c728535..5e5e55837c 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs @@ -72,6 +72,8 @@ public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string /// public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedPrefix))] string expectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.StartsWith"); + CheckParameterNotNull(value, "Assert.StartsWith", "value"); CheckParameterNotNull(expectedPrefix, "Assert.StartsWith", "expectedPrefix"); if (!value.StartsWith(expectedPrefix, comparisonType)) @@ -144,6 +146,8 @@ public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNul /// public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedPrefix))] string notExpectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotStartWith"); + CheckParameterNotNull(value, "Assert.DoesNotStartWith", "value"); CheckParameterNotNull(notExpectedPrefix, "Assert.DoesNotStartWith", "notExpectedPrefix"); if (value.StartsWith(notExpectedPrefix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.That.cs b/src/TestFramework/TestFramework/Assertions/Assert.That.cs index e13c803967..077de1009a 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.That.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.That.cs @@ -26,6 +26,8 @@ public static partial class AssertExtensions /// Thrown if the evaluated condition is . public static void That(Expression> condition, string? message = null, [CallerArgumentExpression(nameof(condition))] string? conditionExpression = null) { + TelemetryCollector.TrackAssertionCall("Assert.That"); + if (condition == null) { throw new ArgumentNullException(nameof(condition)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs index b5478a28a6..83e9960958 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs @@ -321,6 +321,8 @@ public static TException ThrowsExactly(Func action, [Interp private static TException ThrowsException(Action action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(message); @@ -341,6 +343,8 @@ private static TException ThrowsException(Action action, bool isStri private static TException ThrowsException(Action action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(messageBuilder); @@ -477,6 +481,8 @@ public static Task ThrowsExactlyAsync(Func action, private static async Task ThrowsExceptionAsync(Func action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(message); @@ -497,6 +503,8 @@ private static async Task ThrowsExceptionAsync(Func ThrowsExceptionAsync(Func action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(messageBuilder); diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs index c8f37e098b..8fa5356cf7 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs @@ -69,6 +69,8 @@ public static void Contains([NotNull] ICollection? collection, object? element) /// public static void Contains([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.Contains"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.Contains", "collection"); foreach (object? current in collection) @@ -120,6 +122,8 @@ public static void DoesNotContain([NotNull] ICollection? collection, object? ele /// public static void DoesNotContain([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.DoesNotContain"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.DoesNotContain", "collection"); foreach (object? current in collection) @@ -160,6 +164,8 @@ public static void AllItemsAreNotNull([NotNull] ICollection? collection) /// public static void AllItemsAreNotNull([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreNotNull"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreNotNull", "collection"); foreach (object? current in collection) { @@ -202,6 +208,8 @@ public static void AllItemsAreUnique([NotNull] ICollection? collection) /// public static void AllItemsAreUnique([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreUnique"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreUnique", "collection"); message = Assert.ReplaceNulls(message); @@ -292,6 +300,8 @@ public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollecti /// public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); @@ -352,6 +362,8 @@ public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] IColle /// public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsNotSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsNotSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsNotSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); @@ -471,6 +483,8 @@ public static void AreEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? expected, [NotNullIfNotNull(nameof(expected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. @@ -637,6 +651,8 @@ public static void AreNotEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? notExpected, [NotNullIfNotNull(nameof(notExpected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. @@ -740,6 +756,8 @@ public static void AllItemsAreInstancesOfType([NotNull] ICollection? collection, public static void AllItemsAreInstancesOfType( [NotNull] ICollection? collection, [NotNull] Type? expectedType, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreInstancesOfType"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreInstancesOfType", "collection"); Assert.CheckParameterNotNull(expectedType, "CollectionAssert.AllItemsAreInstancesOfType", "expectedType"); int i = 0; @@ -813,6 +831,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual) /// public static void AreEqual(ICollection? expected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, new ObjectComparer(), ref reason)) { @@ -867,6 +887,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual) /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, new ObjectComparer(), ref reason)) { @@ -925,6 +947,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull /// public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, comparer, ref reason)) { @@ -983,6 +1007,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [N /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, comparer, ref reason)) { diff --git a/src/TestFramework/TestFramework/Assertions/StringAssert.cs b/src/TestFramework/TestFramework/Assertions/StringAssert.cs index b2e36625ec..41007ada49 100644 --- a/src/TestFramework/TestFramework/Assertions/StringAssert.cs +++ b/src/TestFramework/TestFramework/Assertions/StringAssert.cs @@ -116,6 +116,8 @@ public static void Contains([NotNull] string? value, [NotNull] string? substring /// public static void Contains([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Contains"); + Assert.CheckParameterNotNull(value, "StringAssert.Contains", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.Contains", "substring"); if (value.IndexOf(substring, comparisonType) < 0) @@ -213,6 +215,8 @@ public static void StartsWith([NotNull] string? value, [NotNull] string? substri /// public static void StartsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.StartsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.StartsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.StartsWith", "substring"); if (!value.StartsWith(substring, comparisonType)) @@ -310,6 +314,8 @@ public static void EndsWith([NotNull] string? value, [NotNull] string? substring /// public static void EndsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.EndsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.EndsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.EndsWith", "substring"); if (!value.EndsWith(substring, comparisonType)) @@ -364,6 +370,8 @@ public static void Matches([NotNull] string? value, [NotNull] Regex? pattern) /// public static void Matches([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Matches"); + Assert.CheckParameterNotNull(value, "StringAssert.Matches", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.Matches", "pattern"); @@ -415,6 +423,8 @@ public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? patter /// public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.DoesNotMatch"); + Assert.CheckParameterNotNull(value, "StringAssert.DoesNotMatch", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.DoesNotMatch", "pattern"); diff --git a/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs new file mode 100644 index 0000000000..209a90e804 --- /dev/null +++ b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Collects aggregated telemetry data about MSTest API usage within a test session. +/// This data is used to understand which APIs are heavily used or unused to guide future investment. +/// +internal static class TelemetryCollector +{ + private static ConcurrentDictionary s_assertionCallCounts = new(); + + /// + /// Records that an assertion method was called. + /// + /// The full name of the assertion (e.g. "Assert.AreEqual", "CollectionAssert.Contains"). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void TrackAssertionCall(string assertionName) + => s_assertionCallCounts.AddOrUpdate(assertionName, 1, static (_, count) => count + 1); + + /// + /// Gets a snapshot of all assertion call counts and resets the counters. + /// This is thread-safe but best-effort: it atomically swaps the dictionary and copies the old one. + /// In-flight calls to that race with the swap may be lost. + /// This is acceptable for telemetry where approximate counts are sufficient. + /// + /// A dictionary mapping assertion names to their (best-effort) call counts. + internal static Dictionary DrainAssertionCallCounts() + { + ConcurrentDictionary old = Interlocked.Exchange(ref s_assertionCallCounts, new ConcurrentDictionary()); +#pragma warning disable IDE0028 // Simplify collection initialization - ConcurrentDictionary snapshot copy + return new Dictionary(old); +#pragma warning restore IDE0028 // Simplify collection initialization + } +} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs new file mode 100644 index 0000000000..f4b4080368 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class TelemetryTests : AcceptanceTestBase +{ + private const string MTPAssetName = "TelemetryMTPProject"; + private const string VSTestAssetName = "TelemetryVSTestProject"; + private const string TestResultsFolderName = "TestResults"; + + #region MTP mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_RunTests_SendsTelemetryWithSettingsAndAttributes(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit +.+mstest\.setting\.parallelization_enabled +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + // Verify attribute usage and config source are also present + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.attribute_usage"), $"Expected attribute_usage in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.config_source"), $"Expected config_source in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.assertion_usage"), $"Expected assertion_usage in telemetry.\n{content}"); + } + + #endregion + + #region MTP mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_DiscoverTests_SendsTelemetryEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--list-tests --diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit[\s\S]+?mstest\.attribute_usage +"""; + await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + } + + #endregion + + #region MTP mode - Telemetry disabled + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_WhenTelemetryDisabled_DoesNotSendMSTestEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + new Dictionary + { + { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" }, + }, + disableTelemetry: false, + TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + + string diagContentsPattern = +""" +.+ Microsoft.Testing.Platform.Telemetry.TelemetryManager DEBUG Telemetry is 'DISABLED' +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsFalse( + Regex.IsMatch(content, "Send telemetry event: dotnet/testingplatform/mstest/sessionexit"), + "MSTest telemetry event should not be sent when telemetry is disabled."); + } + + #endregion + + #region VSTest mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_RunTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --no-build --framework {tfm}", + AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: AssetFixture.VSTestProjectPath, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("Passed!"); + } + + #endregion + + #region VSTest mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_DiscoverTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --no-build --framework {tfm} --list-tests", + AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: AssetFixture.VSTestProjectPath, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test --list-tests failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("PassingTest"); + testResult.AssertOutputContains("DataDrivenTest"); + testResult.AssertOutputContains("TestWithTimeout"); + } + + #endregion + + #region Helpers + + private static async Task AssertDiagnosticReportAsync(TestHostResult testHostResult, string diagPathPattern, string diagContentsPattern, string level = "Trace", string flushType = "async") + { + string outputPattern = $""" +Diagnostic file \(level '{level}' with {flushType} flush\): {diagPathPattern} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + Match match = Regex.Match(testHostResult.StandardOutput, diagPathPattern); + Assert.IsTrue(match.Success, $"{testHostResult}\n{diagPathPattern}"); + + (bool success, string content) = await CheckDiagnosticContentsMatchAsync(match.Value, diagContentsPattern); + Assert.IsTrue(success, $"{content}\n{diagContentsPattern}"); + + return match.Value; + } + + private static async Task<(bool IsMatch, string Content)> CheckDiagnosticContentsMatchAsync(string path, string pattern) + { + using var reader = new StreamReader(path); + string content = await reader.ReadToEndAsync(); + + return (Regex.IsMatch(content, pattern), content); + } + + #endregion + + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) + { + private const string MTPProjectId = nameof(TelemetryTests) + "_MTP"; + private const string VSTestProjectId = nameof(TelemetryTests) + "_VSTest"; + + public string MTPProjectPath => GetAssetPath(MTPProjectId); + + public string VSTestProjectPath => GetAssetPath(VSTestProjectId); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + // MTP mode project (MSTest runner enabled, no global.json override) + yield return (MTPProjectId, MTPAssetName, + MTPSourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + // VSTest mode project (uses global.json to force VSTest runner) + yield return (VSTestProjectId, VSTestAssetName, + VSTestSourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion)); + } + + private const string MTPSourceCode = """ +#file TelemetryMTPProject.csproj + + + + Exe + true + $TargetFrameworks$ + Preview + + + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} +"""; + + private const string VSTestSourceCode = """ +#file TelemetryVSTestProject.csproj + + + + $TargetFrameworks$ + Preview + false + true + + + + + + + + + + +#file global.json +{ + "test": { + "runner": "VSTest" + } +} + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} +"""; + } + + public TestContext TestContext { get; set; } +}