From 158cba5a1325de9452f79203a5f6991496791e40 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 27 Dec 2024 09:50:18 +0100 Subject: [PATCH 01/11] Initial work for ignoring specific test cases --- .../Discovery/AssemblyEnumerator.cs | 2 +- .../Discovery/TypeEnumerator.cs | 1 - .../MSTest.TestAdapter/Execution/TestClassInfo.cs | 6 ++++-- .../Execution/UnitTestRunner.cs | 10 +++++++++- .../ObjectModel/UnitTestElement.cs | 5 ----- .../Attributes/DataSource/DataRowAttribute.cs | 7 ++++++- .../Attributes/DataSource/DynamicDataAttribute.cs | 7 ++++++- .../Attributes/TestMethod/TestClassAttribute.cs | 5 +++++ .../Attributes/TestMethod/TestMethodAttribute.cs | 5 +++++ .../Interfaces/ITestDataSourceIgnoreCapability.cs | 15 +++++++++++++++ .../PublicAPI/PublicAPI.Unshipped.txt | 11 +++++++++++ .../Discovery/TypeEnumeratorTests.cs | 12 ------------ .../Execution/TestExecutionManagerTests.cs | 9 +++------ 13 files changed, 65 insertions(+), 30 deletions(-) create mode 100644 src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 884fe813fb..1144be3949 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -342,7 +342,6 @@ static UnitTestElement GetFixtureTest(string classFullName, string assemblyLocat return new UnitTestElement(method) { DisplayName = $"[{fixtureType}] {methodName}", - Ignored = true, Traits = [new Trait(Constants.FixturesTestTrait, fixtureType)], }; } @@ -424,6 +423,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests. // Any change made here should be reflected in TestMethodRunner.ExecuteDataSourceBasedTests as well. data = dataSource.GetData(methodInfo); + string? ignoreReason = (dataSource as ITestDataSourceIgnoreCapability)?.Ignore; if (!data.Any()) { diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs index 07d9327abd..cbb50767f7 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/TypeEnumerator.cs @@ -155,7 +155,6 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT TestCategory = _reflectHelper.GetTestCategories(method, _type), DoNotParallelize = _reflectHelper.IsDoNotParallelizeSet(method, _type), Priority = _reflectHelper.GetPriority(method), - Ignored = _reflectHelper.IsNonDerivedAttributeDefined(method, inherit: false), DeploymentItems = PlatformServiceProvider.Instance.TestDeployment.GetDeploymentItems(method, _type, warnings), }; diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs index 417b507c24..b8162d7752 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs @@ -568,7 +568,8 @@ internal void ExecuteClassCleanup(TestContext testContext) { if (classCleanupMethod is not null) { - if (!ReflectHelper.Instance.IsNonDerivedAttributeDefined(classCleanupMethod.DeclaringType!, false)) + if (ClassAttribute.Ignore is null && + !ReflectHelper.Instance.IsNonDerivedAttributeDefined(classCleanupMethod.DeclaringType!, false)) { ClassCleanupException = InvokeCleanupMethod(classCleanupMethod, remainingCleanupCount: BaseClassCleanupMethods.Count, testContext); } @@ -579,7 +580,8 @@ internal void ExecuteClassCleanup(TestContext testContext) for (int i = 0; i < BaseClassCleanupMethods.Count; i++) { classCleanupMethod = BaseClassCleanupMethods[i]; - if (!ReflectHelper.Instance.IsNonDerivedAttributeDefined(classCleanupMethod.DeclaringType!, false)) + if (ClassAttribute.Ignore is null && + !ReflectHelper.Instance.IsNonDerivedAttributeDefined(classCleanupMethod.DeclaringType!, false)) { ClassCleanupException = InvokeCleanupMethod(classCleanupMethod, remainingCleanupCount: BaseClassCleanupMethods.Count - 1 - i, testContext); if (ClassCleanupException is not null) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index acfd86fdb3..30ac3971a6 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -347,9 +347,17 @@ private bool IsTestMethodRunnable( } } + // TODO: Executor should never be null. Is it incorrectly annotated? + string? ignoreMessage = testMethodInfo.TestMethodOptions.Executor?.Ignore ?? testMethodInfo.Parent.ClassAttribute.Ignore; + if (ignoreMessage is not null) + { + notRunnableResult = [new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage)]; + return false; + } + IgnoreAttribute? ignoreAttributeOnClass = _reflectHelper.GetFirstNonDerivedAttributeOrDefault(testMethodInfo.Parent.ClassType, inherit: false); - string? ignoreMessage = ignoreAttributeOnClass?.IgnoreMessage; + ignoreMessage = ignoreAttributeOnClass?.IgnoreMessage; IgnoreAttribute? ignoreAttributeOnMethod = _reflectHelper.GetFirstNonDerivedAttributeOrDefault(testMethodInfo.TestMethod, inherit: false); diff --git a/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs b/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs index a585c6afde..e180078f4b 100644 --- a/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs +++ b/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs @@ -33,11 +33,6 @@ public UnitTestElement(TestMethod testMethod) /// public TestMethod TestMethod { get; private set; } - /// - /// Gets or sets a value indicating whether the unit test should be ignored at run-time. - /// - public bool Ignored { get; set; } - /// /// Gets or sets a value indicating whether it is a async test. /// diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs index 4fdb946813..03720be62f 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// Attribute to define in-line data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability +public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability, ITestDataSourceIgnoreCapability { /// /// Initializes a new instance of the class. @@ -53,6 +53,11 @@ public DataRowAttribute(string?[]? stringArrayData) /// public string? DisplayName { get; set; } + /// + /// Gets or sets a reason to ignore the specific test case. Setting the property to non-null value will ignore the test case. + /// + public string? Ignore { get; set; } + /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs index 48825c992c..08aa54803f 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs @@ -32,7 +32,7 @@ public enum DynamicDataSourceType /// Attribute to define dynamic data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability +public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability, ITestDataSourceIgnoreCapability { private readonly string _dynamicDataSourceName; private readonly DynamicDataSourceType _dynamicDataSourceType; @@ -114,6 +114,11 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; + /// + /// Gets or sets a reason to ignore this dynamic data source. Setting the property to non-null value will ignore the dynamic data source. + /// + public string? Ignore { get; set; } + /// public IEnumerable GetData(MethodInfo methodInfo) => DynamicDataProvider.Instance.GetData(_dynamicDataDeclaringType, _dynamicDataSourceType, _dynamicDataSourceName, methodInfo); diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs index da57c79550..454dc8d5c4 100644 --- a/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs @@ -26,4 +26,9 @@ public class TestClassAttribute : Attribute public virtual TestMethodAttribute? GetTestMethodAttribute(TestMethodAttribute? testMethodAttribute) => // If TestMethod is not extended by derived class then return back the original TestMethodAttribute testMethodAttribute; + + /// + /// Gets or sets a reason to ignore the test class. Setting the property to non-null value will ignore the test class. + /// + public string? Ignore { get; set; } } diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs index 319fcec986..b6842ec0bf 100644 --- a/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs @@ -50,6 +50,11 @@ public TestMethodAttribute() /// public string? DisplayName { get; } + /// + /// Gets or sets a reason to ignore the test method. Setting the property to non-null value will ignore the test method. + /// + public string? Ignore { get; set; } + /// /// Executes a test method. /// diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs new file mode 100644 index 0000000000..07ff45f5db --- /dev/null +++ b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Specifies the capability of a test data source to be ignored and define the ignore reason. +/// +public interface ITestDataSourceIgnoreCapability +{ + /// + /// Gets or sets a reason to ignore the test data source. Setting the property to non-null value will ignore the test data source. + /// + string? Ignore { get; set; } +} diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index 3ceaf8e88e..6eee3a9e92 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -1,11 +1,22 @@ #nullable enable +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.Ignore.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.Ignore.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.Ignore.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.Ignore.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.Ignore.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.Ignore.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.Ignore.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.Ignore.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.Ignore.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.Ignore.set -> void static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Throws(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsAsync(System.Func! action, string! message = "", params object![]! messageArgs) -> System.Threading.Tasks.Task! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactly(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs index 60f85dff0e..5921ef972e 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Discovery/TypeEnumeratorTests.cs @@ -291,18 +291,6 @@ public void GetTestFromMethodShouldInitializeAsyncTypeNameCorrectly() Verify(expectedAsyncTaskName == testElement.AsyncTypeName); } - public void GetTestFromMethodShouldSetIgnoredPropertyToFalseIfNotSetOnTestClassAndTestMethod() - { - SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameAssembly: true); - TypeEnumerator typeEnumerator = GetTypeEnumeratorInstance(typeof(DummyTestClass), "DummyAssemblyName"); - MethodInfo methodInfo = typeof(DummyTestClass).GetMethod("MethodWithVoidReturnType"); - - MSTest.TestAdapter.ObjectModel.UnitTestElement testElement = typeEnumerator.GetTestFromMethod(methodInfo, true, _warnings); - - Verify(testElement is not null); - Verify(!testElement.Ignored); - } - public void GetTestFromMethodShouldSetTestCategory() { SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameAssembly: true); diff --git a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs index bd6cf90af7..f00010e440 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs +++ b/test/UnitTests/MSTestAdapter.UnitTests/Execution/TestExecutionManagerTests.cs @@ -123,7 +123,7 @@ public void RunTestsForTestWithFilterShouldSendResultsForFilteredTests() public void RunTestsForIgnoredTestShouldSendResultsMarkingIgnoredTestsAsSkipped() { - TestCase testCase = GetTestCase(typeof(DummyTestClass), "IgnoredTest", ignore: true); + TestCase testCase = GetTestCase(typeof(DummyTestClass), "IgnoredTest"); TestCase[] tests = [testCase]; _testExecutionManager.RunTests(tests, _runContext, _frameworkHandle, _cancellationToken); @@ -814,14 +814,11 @@ private void RunTestsForTestShouldRunTestsInTheParentDomainsApartmentState() #region private methods - private static TestCase GetTestCase(Type typeOfClass, string testName, bool ignore = false) + private static TestCase GetTestCase(Type typeOfClass, string testName) { MethodInfo methodInfo = typeOfClass.GetMethod(testName); var testMethod = new TestMethod(methodInfo.Name, typeOfClass.FullName, Assembly.GetExecutingAssembly().Location, isAsync: false); - UnitTestElement element = new(testMethod) - { - Ignored = ignore, - }; + UnitTestElement element = new(testMethod); return element.ToTestCase(); } From 1bae8e752e36705da7bc91420e00d1e8f004c38d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 27 Dec 2024 10:21:07 +0100 Subject: [PATCH 02/11] Progress --- src/Adapter/MSTest.TestAdapter/Constants.cs | 2 ++ .../Discovery/AssemblyEnumerator.cs | 4 +++- .../Execution/TestMethodRunner.cs | 19 +++++++++++++++++++ .../Extensions/TestCaseExtensions.cs | 1 + .../Extensions/TestResultExtensions.cs | 10 +++++++++- .../ObjectModel/TestMethod.cs | 2 ++ .../ObjectModel/UnitTestElement.cs | 1 + .../Attributes/TestMethod/TestResult.cs | 2 ++ 8 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Constants.cs b/src/Adapter/MSTest.TestAdapter/Constants.cs index f110859dc8..af2738fc9e 100644 --- a/src/Adapter/MSTest.TestAdapter/Constants.cs +++ b/src/Adapter/MSTest.TestAdapter/Constants.cs @@ -123,6 +123,8 @@ internal static class Constants internal static readonly TestProperty TestDynamicDataProperty = TestProperty.Register("MSTest.DynamicData", "DynamicData", typeof(string[]), TestPropertyAttributes.Hidden, typeof(TestCase)); internal static readonly TestProperty TestIdGenerationStrategyProperty = TestProperty.Register("MSTest.TestIdGenerationStrategy", "TestIdGenerationStrategy", typeof(int), TestPropertyAttributes.Hidden, typeof(TestCase)); + + internal static readonly TestProperty TestDataSourceIgnoreReasonProperty = TestProperty.Register("MSTest.TestDataSourceIgnoreReasonProperty", "TestDataSourceIgnoreReasonProperty", typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase)); #endregion #region Private Constants diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 1144be3949..21321b9e01 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -423,7 +423,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests. // Any change made here should be reflected in TestMethodRunner.ExecuteDataSourceBasedTests as well. data = dataSource.GetData(methodInfo); - string? ignoreReason = (dataSource as ITestDataSourceIgnoreCapability)?.Ignore; + string? testDataSourceIgnoreReason = (dataSource as ITestDataSourceIgnoreCapability)?.Ignore; if (!data.Any()) { @@ -435,6 +435,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat UnitTestElement discoveredTest = test.Clone(); // Make the test not data driven, because it had no data. discoveredTest.TestMethod.DataType = DynamicDataType.None; + discoveredTest.TestMethod.TestDataSourceIgnoreReason = testDataSourceIgnoreReason; discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName; tests.Add(discoveredTest); @@ -468,6 +469,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat try { discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); + discoveredTest.TestMethod.TestDataSourceIgnoreReason = testDataSourceIgnoreReason; discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; } catch (SerializationException ex) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index a3ee59fa98..a3f52e67d9 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -168,6 +168,11 @@ internal UnitTestResult[] RunTestMethod() { if (_test.DataType == DynamicDataType.ITestDataSource) { + if (_test.TestDataSourceIgnoreReason is not null) + { + return [new(UnitTestOutcome.Ignored, _test.TestDataSourceIgnoreReason)]; + } + object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData); TestResult[] testResults = ExecuteTestWithDataSource(null, data); results.AddRange(testResults); @@ -306,6 +311,20 @@ private bool ExecuteDataSourceBasedTests(List results) foreach (UTF.ITestDataSource testDataSource in testDataSources) { isDataDriven = true; + + if (testDataSource is ITestDataSourceIgnoreCapability { Ignore: { } ignoreReason }) + { + results.Add(new() + { + // This is closest to ignore. This enum doesn't have a value specific to Ignore. + // It may be a better idea to add a value there, but the enum is public and we need to think more carefully before adding the API. + // For now, TestResultExtensions.ToUnitTestResults method will convert this to Ignored value of ObjectModel enum when IgnoreReason is non-null. + Outcome = UTF.UnitTestOutcome.NotRunnable, + IgnoreReason = ignoreReason, + }); + continue; + } + IEnumerable? dataSource; // This code is to execute tests. To discover the tests code is in AssemblyEnumerator.ProcessTestDataSourceTests. diff --git a/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs b/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs index c0ed82d9cb..3e8b0207a8 100644 --- a/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs @@ -86,6 +86,7 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string testMethod.DataType = dataType; testMethod.SerializedData = data; + testMethod.TestDataSourceIgnoreReason = testCase.GetPropertyValue(Constants.TestDataSourceIgnoreReasonProperty) as string; } if (testCase.GetPropertyValue(Constants.DeclaringClassNameProperty) is string declaringClassName && declaringClassName != testClassName) diff --git a/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs b/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs index 301984dbb1..a600eeb218 100644 --- a/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/Extensions/TestResultExtensions.cs @@ -29,7 +29,9 @@ internal static UnitTestResult[] ToUnitTestResults(this IReadOnlyCollection internal string?[]? SerializedData { get; set; } + internal string? TestDataSourceIgnoreReason { get; set; } + /// /// Gets or sets the test group set during discovery. /// diff --git a/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs b/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs index e180078f4b..35aab5c9e1 100644 --- a/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs +++ b/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs @@ -206,6 +206,7 @@ internal TestCase ToTestCase() testCase.SetPropertyValue(Constants.TestDynamicDataTypeProperty, (int)TestMethod.DataType); testCase.SetPropertyValue(Constants.TestDynamicDataProperty, data); + testCase.SetPropertyValue(Constants.TestDataSourceIgnoreReasonProperty, TestMethod.TestDataSourceIgnoreReason); } SetTestCaseId(testCase, testFullName); diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/TestResult.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/TestResult.cs index 1b0b35fe7b..0ccf2e316c 100644 --- a/src/TestFramework/TestFramework/Attributes/TestMethod/TestResult.cs +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/TestResult.cs @@ -24,6 +24,8 @@ public class TestResult /// public UnitTestOutcome Outcome { get; set; } + internal string? IgnoreReason { get; set; } + /// /// Gets or sets the exception thrown when test is failed. /// From 4fa10cb44cdb461b280e8e365f49ef17af7639dc Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 10:52:49 +0100 Subject: [PATCH 03/11] Rename Ignore to IgnoreMessage --- src/Adapter/MSTest.TestAdapter/Constants.cs | 2 +- .../Discovery/AssemblyEnumerator.cs | 6 +++--- .../Execution/TestClassInfo.cs | 4 ++-- .../Execution/TestMethodRunner.cs | 6 +++--- .../Execution/UnitTestRunner.cs | 2 +- .../Extensions/TestCaseExtensions.cs | 2 +- .../ObjectModel/TestMethod.cs | 2 +- .../ObjectModel/UnitTestElement.cs | 2 +- .../Attributes/DataSource/DataRowAttribute.cs | 2 +- .../DataSource/DynamicDataAttribute.cs | 2 +- .../TestMethod/TestClassAttribute.cs | 2 +- .../TestMethod/TestMethodAttribute.cs | 2 +- .../ITestDataSourceIgnoreCapability.cs | 2 +- .../PublicAPI/PublicAPI.Unshipped.txt | 20 +++++++++---------- 14 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Constants.cs b/src/Adapter/MSTest.TestAdapter/Constants.cs index af2738fc9e..569d49159b 100644 --- a/src/Adapter/MSTest.TestAdapter/Constants.cs +++ b/src/Adapter/MSTest.TestAdapter/Constants.cs @@ -124,7 +124,7 @@ internal static class Constants internal static readonly TestProperty TestIdGenerationStrategyProperty = TestProperty.Register("MSTest.TestIdGenerationStrategy", "TestIdGenerationStrategy", typeof(int), TestPropertyAttributes.Hidden, typeof(TestCase)); - internal static readonly TestProperty TestDataSourceIgnoreReasonProperty = TestProperty.Register("MSTest.TestDataSourceIgnoreReasonProperty", "TestDataSourceIgnoreReasonProperty", typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase)); + internal static readonly TestProperty TestDataSourceIgnoreMessageProperty = TestProperty.Register("MSTest.TestDataSourceIgnoreMessageProperty", "TestDataSourceIgnoreMessageProperty", typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase)); #endregion #region Private Constants diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 21321b9e01..74654b6bca 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -423,7 +423,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat // This code is to discover tests. To run the tests code is in TestMethodRunner.ExecuteDataSourceBasedTests. // Any change made here should be reflected in TestMethodRunner.ExecuteDataSourceBasedTests as well. data = dataSource.GetData(methodInfo); - string? testDataSourceIgnoreReason = (dataSource as ITestDataSourceIgnoreCapability)?.Ignore; + string? testDataSourceIgnoreMessage = (dataSource as ITestDataSourceIgnoreCapability)?.IgnoreMessage; if (!data.Any()) { @@ -435,7 +435,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat UnitTestElement discoveredTest = test.Clone(); // Make the test not data driven, because it had no data. discoveredTest.TestMethod.DataType = DynamicDataType.None; - discoveredTest.TestMethod.TestDataSourceIgnoreReason = testDataSourceIgnoreReason; + discoveredTest.TestMethod.TestDataSourceIgnoreMessage = testDataSourceIgnoreMessage; discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName; tests.Add(discoveredTest); @@ -469,7 +469,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat try { discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); - discoveredTest.TestMethod.TestDataSourceIgnoreReason = testDataSourceIgnoreReason; + discoveredTest.TestMethod.TestDataSourceIgnoreMessage = testDataSourceIgnoreMessage; discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; } catch (SerializationException ex) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs index b8162d7752..8c2ee5b10f 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs @@ -568,7 +568,7 @@ internal void ExecuteClassCleanup(TestContext testContext) { if (classCleanupMethod is not null) { - if (ClassAttribute.Ignore is null && + if (ClassAttribute.IgnoreMessage is null && !ReflectHelper.Instance.IsNonDerivedAttributeDefined(classCleanupMethod.DeclaringType!, false)) { ClassCleanupException = InvokeCleanupMethod(classCleanupMethod, remainingCleanupCount: BaseClassCleanupMethods.Count, testContext); @@ -580,7 +580,7 @@ internal void ExecuteClassCleanup(TestContext testContext) for (int i = 0; i < BaseClassCleanupMethods.Count; i++) { classCleanupMethod = BaseClassCleanupMethods[i]; - if (ClassAttribute.Ignore is null && + if (ClassAttribute.IgnoreMessage is null && !ReflectHelper.Instance.IsNonDerivedAttributeDefined(classCleanupMethod.DeclaringType!, false)) { ClassCleanupException = InvokeCleanupMethod(classCleanupMethod, remainingCleanupCount: BaseClassCleanupMethods.Count - 1 - i, testContext); diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index a3f52e67d9..cb3f358b30 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -168,9 +168,9 @@ internal UnitTestResult[] RunTestMethod() { if (_test.DataType == DynamicDataType.ITestDataSource) { - if (_test.TestDataSourceIgnoreReason is not null) + if (_test.TestDataSourceIgnoreMessage is not null) { - return [new(UnitTestOutcome.Ignored, _test.TestDataSourceIgnoreReason)]; + return [new(UnitTestOutcome.Ignored, _test.TestDataSourceIgnoreMessage)]; } object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData); @@ -312,7 +312,7 @@ private bool ExecuteDataSourceBasedTests(List results) { isDataDriven = true; - if (testDataSource is ITestDataSourceIgnoreCapability { Ignore: { } ignoreReason }) + if (testDataSource is ITestDataSourceIgnoreCapability { IgnoreMessage: { } ignoreReason }) { results.Add(new() { diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index 30ac3971a6..7b1d1ef86d 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -348,7 +348,7 @@ private bool IsTestMethodRunnable( } // TODO: Executor should never be null. Is it incorrectly annotated? - string? ignoreMessage = testMethodInfo.TestMethodOptions.Executor?.Ignore ?? testMethodInfo.Parent.ClassAttribute.Ignore; + string? ignoreMessage = testMethodInfo.TestMethodOptions.Executor?.IgnoreMessage ?? testMethodInfo.Parent.ClassAttribute.IgnoreMessage; if (ignoreMessage is not null) { notRunnableResult = [new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage)]; diff --git a/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs b/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs index 3e8b0207a8..fafe4e6346 100644 --- a/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs +++ b/src/Adapter/MSTest.TestAdapter/Extensions/TestCaseExtensions.cs @@ -86,7 +86,7 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string testMethod.DataType = dataType; testMethod.SerializedData = data; - testMethod.TestDataSourceIgnoreReason = testCase.GetPropertyValue(Constants.TestDataSourceIgnoreReasonProperty) as string; + testMethod.TestDataSourceIgnoreMessage = testCase.GetPropertyValue(Constants.TestDataSourceIgnoreMessageProperty) as string; } if (testCase.GetPropertyValue(Constants.DeclaringClassNameProperty) is string declaringClassName && declaringClassName != testClassName) diff --git a/src/Adapter/MSTest.TestAdapter/ObjectModel/TestMethod.cs b/src/Adapter/MSTest.TestAdapter/ObjectModel/TestMethod.cs index 0e3c804169..5b94d2bfa5 100644 --- a/src/Adapter/MSTest.TestAdapter/ObjectModel/TestMethod.cs +++ b/src/Adapter/MSTest.TestAdapter/ObjectModel/TestMethod.cs @@ -143,7 +143,7 @@ public string? DeclaringClassFullName /// internal string?[]? SerializedData { get; set; } - internal string? TestDataSourceIgnoreReason { get; set; } + internal string? TestDataSourceIgnoreMessage { get; set; } /// /// Gets or sets the test group set during discovery. diff --git a/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs b/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs index 35aab5c9e1..1701b39c82 100644 --- a/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs +++ b/src/Adapter/MSTest.TestAdapter/ObjectModel/UnitTestElement.cs @@ -206,7 +206,7 @@ internal TestCase ToTestCase() testCase.SetPropertyValue(Constants.TestDynamicDataTypeProperty, (int)TestMethod.DataType); testCase.SetPropertyValue(Constants.TestDynamicDataProperty, data); - testCase.SetPropertyValue(Constants.TestDataSourceIgnoreReasonProperty, TestMethod.TestDataSourceIgnoreReason); + testCase.SetPropertyValue(Constants.TestDataSourceIgnoreMessageProperty, TestMethod.TestDataSourceIgnoreMessage); } SetTestCaseId(testCase, testFullName); diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs index 03720be62f..040c794cf4 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs @@ -56,7 +56,7 @@ public DataRowAttribute(string?[]? stringArrayData) /// /// Gets or sets a reason to ignore the specific test case. Setting the property to non-null value will ignore the test case. /// - public string? Ignore { get; set; } + public string? IgnoreMessage { get; set; } /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs index 08aa54803f..370ab10af2 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs @@ -117,7 +117,7 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar /// /// Gets or sets a reason to ignore this dynamic data source. Setting the property to non-null value will ignore the dynamic data source. /// - public string? Ignore { get; set; } + public string? IgnoreMessage { get; set; } /// public IEnumerable GetData(MethodInfo methodInfo) diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs index 454dc8d5c4..0da97a08d2 100644 --- a/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/TestClassAttribute.cs @@ -30,5 +30,5 @@ public class TestClassAttribute : Attribute /// /// Gets or sets a reason to ignore the test class. Setting the property to non-null value will ignore the test class. /// - public string? Ignore { get; set; } + public string? IgnoreMessage { get; set; } } diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs index b6842ec0bf..1d42b34d43 100644 --- a/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/TestMethodAttribute.cs @@ -53,7 +53,7 @@ public TestMethodAttribute() /// /// Gets or sets a reason to ignore the test method. Setting the property to non-null value will ignore the test method. /// - public string? Ignore { get; set; } + public string? IgnoreMessage { get; set; } /// /// Executes a test method. diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs index 07ff45f5db..0a22b7dc5f 100644 --- a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs +++ b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceIgnoreCapability.cs @@ -11,5 +11,5 @@ public interface ITestDataSourceIgnoreCapability /// /// Gets or sets a reason to ignore the test data source. Setting the property to non-null value will ignore the test data source. /// - string? Ignore { get; set; } + string? IgnoreMessage { get; set; } } diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index 6eee3a9e92..cc571578d3 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -1,22 +1,22 @@ #nullable enable -Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.Ignore.get -> string? -Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.Ignore.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IgnoreMessage.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IgnoreMessage.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void -Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.Ignore.get -> string? -Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.Ignore.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.IgnoreMessage.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.IgnoreMessage.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability -Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.Ignore.get -> string? -Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.Ignore.set -> void -Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.Ignore.get -> string? -Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.Ignore.set -> void -Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.Ignore.get -> string? -Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.Ignore.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.IgnoreMessage.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSourceIgnoreCapability.IgnoreMessage.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute.IgnoreMessage.set -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.IgnoreMessage.get -> string? +Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute.IgnoreMessage.set -> void static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Throws(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsAsync(System.Func! action, string! message = "", params object![]! messageArgs) -> System.Threading.Tasks.Task! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactly(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! From bbae3995bdc3e07c313a248101b2f75c74e4fb53 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 10:54:28 +0100 Subject: [PATCH 04/11] Check class ignore first --- src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs index 7b1d1ef86d..611e238ec5 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs @@ -348,7 +348,7 @@ private bool IsTestMethodRunnable( } // TODO: Executor should never be null. Is it incorrectly annotated? - string? ignoreMessage = testMethodInfo.TestMethodOptions.Executor?.IgnoreMessage ?? testMethodInfo.Parent.ClassAttribute.IgnoreMessage; + string? ignoreMessage = testMethodInfo.Parent.ClassAttribute.IgnoreMessage ?? testMethodInfo.TestMethodOptions.Executor?.IgnoreMessage; if (ignoreMessage is not null) { notRunnableResult = [new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage)]; From e9a277e0520a12e1efe7908c49baad5c93e0478f Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 11:06:59 +0100 Subject: [PATCH 05/11] Few basic tests --- .../IgnoreTests.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs index 74d6740d73..8b0ad021aa 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs @@ -3,6 +3,7 @@ using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; namespace MSTest.Acceptance.IntegrationTests; @@ -35,6 +36,28 @@ public async Task WhenAllTestsAreIgnored_AssemblyInitializeAndCleanupAreSkipped( testHostResult.AssertOutputDoesNotContain("AssemblyCleanup"); } + [TestMethod] + public async Task WhenTestClassIsIgnoredViaIgnoreMessageProperty() + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent); + TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings --filter TestClassWithIgnoreMessage"); + + // Assert + testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 1); + } + + [TestMethod] + public async Task WhenTestMethodIsIgnoredViaIgnoreMessageProperty() + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent); + TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings --filter TestClassWithMethodUsingIgnoreMessage"); + + // Assert + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 0, skipped: 1); + } + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) { public const string ProjectName = "TestIgnore"; @@ -146,6 +169,27 @@ public void TestMethod1() { } } + +[TestClass(IgnoreMessage = "This test class is ignored")] +public class TestClassWithIgnoreMessage +{ + [TestMethod] + public void TestMethod1() + { + } +} + +public class TestClassWithMethodUsingIgnoreMessage +{ + [TestMethod(IgnoreMessage = "This test method is ignored")] + public void TestMethod1() + { + } + + public void TestMethod2() + { + } +} """; } } From a494d2b300559e114559f08a0f261871f884d16f Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 11:28:09 +0100 Subject: [PATCH 06/11] Fix tests --- .../MSTest.Acceptance.IntegrationTests/IgnoreTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs index 8b0ad021aa..a414547d94 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs @@ -43,7 +43,7 @@ public async Task WhenTestClassIsIgnoredViaIgnoreMessageProperty() TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings --filter TestClassWithIgnoreMessage"); // Assert - testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 1); } @@ -54,7 +54,7 @@ public async Task WhenTestMethodIsIgnoredViaIgnoreMessageProperty() TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings --filter TestClassWithMethodUsingIgnoreMessage"); // Assert - testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); + testHostResult.AssertExitCodeIs(ExitCodes.Success); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 0, skipped: 1); } @@ -179,6 +179,7 @@ public void TestMethod1() } } +[TestClass] public class TestClassWithMethodUsingIgnoreMessage { [TestMethod(IgnoreMessage = "This test method is ignored")] @@ -186,6 +187,7 @@ public void TestMethod1() { } + [TestMethod] public void TestMethod2() { } From ba23fad985dce35ccf0703edf36de95f2023bdd8 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 11:30:47 +0100 Subject: [PATCH 07/11] Fix tests --- .../MSTest.Acceptance.IntegrationTests/IgnoreTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs index a414547d94..c8b04e51eb 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs @@ -18,7 +18,7 @@ public async Task ClassCleanup_Inheritance_WhenClassIsSkipped() // Assert testHostResult.AssertExitCodeIs(0); - testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 1); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 2, skipped: 3); testHostResult.AssertOutputContains("SubClass.Method"); } @@ -44,7 +44,7 @@ public async Task WhenTestClassIsIgnoredViaIgnoreMessageProperty() // Assert testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); - testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 1); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 0, skipped: 1); } [TestMethod] @@ -55,7 +55,7 @@ public async Task WhenTestMethodIsIgnoredViaIgnoreMessageProperty() // Assert testHostResult.AssertExitCodeIs(ExitCodes.Success); - testHostResult.AssertOutputContainsSummary(failed: 0, passed: 0, skipped: 1); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 1); } public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) From bc9adae122140ac48ab5e40bfde363fae1da7f63 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 11:31:52 +0100 Subject: [PATCH 08/11] Use ExitCodes --- .../MSTest.Acceptance.IntegrationTests/IgnoreTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs index c8b04e51eb..5b710ce54d 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs @@ -17,7 +17,7 @@ public async Task ClassCleanup_Inheritance_WhenClassIsSkipped() TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings --filter ClassName!~TestClassWithAssemblyInitialize"); // Assert - testHostResult.AssertExitCodeIs(0); + testHostResult.AssertExitCodeIs(ExitCodes.Success); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 2, skipped: 3); testHostResult.AssertOutputContains("SubClass.Method"); @@ -30,7 +30,7 @@ public async Task WhenAllTestsAreIgnored_AssemblyInitializeAndCleanupAreSkipped( TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings --filter TestClassWithAssemblyInitialize"); // Assert - testHostResult.AssertExitCodeIs(8); + testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 0, skipped: 1); testHostResult.AssertOutputDoesNotContain("AssemblyInitialize"); testHostResult.AssertOutputDoesNotContain("AssemblyCleanup"); From e9ca88924637c8846280c1adbba1f776205e394a Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 12:17:43 +0100 Subject: [PATCH 09/11] Extra tests --- .../IgnoreTests.cs | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs index 5b710ce54d..c98f093d45 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs @@ -18,7 +18,7 @@ public async Task ClassCleanup_Inheritance_WhenClassIsSkipped() // Assert testHostResult.AssertExitCodeIs(ExitCodes.Success); - testHostResult.AssertOutputContainsSummary(failed: 0, passed: 2, skipped: 3); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 12, skipped: 9); testHostResult.AssertOutputContains("SubClass.Method"); } @@ -58,6 +58,17 @@ public async Task WhenTestMethodIsIgnoredViaIgnoreMessageProperty() testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 1); } + [TestMethod] + public async Task WhenSpecificDataSourceIsIgnoredViaIgnoreMessageProperty() + { + var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent); + TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings --filter TestClassWithDataSourcesUsingIgnoreMessage"); + + // Assert + testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 10, skipped: 6); + } + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) { public const string ProjectName = "TestIgnore"; @@ -104,6 +115,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture. #file UnitTest1.cs using System; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -192,6 +204,50 @@ public void TestMethod2() { } } + +[TestClass] +public class TestClassWithDataSourcesUsingIgnoreMessage +{ + [TestMethod] // 1 skipped, 2 pass + [DataRow(0, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + [DataRow(1, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold, IgnoreMessage = "This data row is ignored")] + [DataRow(2, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + public void TestMethod1(int i) + { + } + + [TestMethod] // 1 skipped (folded), 3 pass + [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold, IgnoreMessage = "This source is ignored")] + public void TestMethod2(int i) + { + } + + [TestMethod] // 1 skipped, 2 pass + [DataRow(0, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold)] + [DataRow(1, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold, IgnoreMessage = "This data row is ignored")] + [DataRow(2, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold)] + public void TestMethod3(int i) + { + } + + [TestMethod] // 3 skipped (unfolded), 3 pass + [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold)] + [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold, IgnoreMessage = "This source is ignored")] + public void TestMethod4(int i) + { + } + + public static IEnumerable Data + { + get + { + yield return new object[] { 0 }; + yield return new object[] { 1 }; + yield return new object[] { 2 }; + } + } +} """; } } From a10d9c3d11dadbf9eff172b05b497a9f7fd9bb36 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 12:43:10 +0100 Subject: [PATCH 10/11] Improve test --- .../IgnoreTests.cs | 86 +++++++++++++++++-- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs index c98f093d45..3d3886f171 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs @@ -66,6 +66,55 @@ public async Task WhenSpecificDataSourceIsIgnoredViaIgnoreMessageProperty() // Assert testHostResult.AssertExitCodeIs(ExitCodes.Success); + testHostResult.AssertOutputMatchesRegex(""" + TestInitialize: TestMethod1 \(0\) + TestCleanup: TestMethod1 \(0\) + TestInitialize: TestMethod1 \(2\) + TestCleanup: TestMethod1 \(2\) + TestInitialize: TestMethod2 \(0\) + TestCleanup: TestMethod2 \(0\) + TestInitialize: TestMethod2 \(1\) + TestCleanup: TestMethod2 \(1\) + TestInitialize: TestMethod2 \(2\) + TestCleanup: TestMethod2 \(2\) + skipped TestMethod1 \(\d+ms\) + skipped TestMethod2 \(\d+ms\) + TestInitialize: TestMethod3 \(0\) + TestCleanup: TestMethod3 \(0\) + skipped TestMethod3 \(1\) \(\d+ms\) + TestInitialize: TestMethod3 \(2\) + TestCleanup: TestMethod3 \(2\) + TestInitialize: TestMethod4 \(0\) + TestCleanup: TestMethod4 \(0\) + TestInitialize: TestMethod4 \(1\) + TestCleanup: TestMethod4 \(1\) + TestInitialize: TestMethod4 \(2\) + TestCleanup: TestMethod4 \(2\) + skipped TestMethod4 \(3\) \(\d+ms\) + skipped TestMethod4 \(4\) \(\d+ms\) + skipped TestMethod4 \(5\) \(\d+ms\) + """); + + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod1 (1)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod1 (1)"); + + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod2 (3)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod2 (3)"); + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod2 (4)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod2 (4)"); + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod2 (5)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod2 (5)"); + + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod3 (1)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod3 (1)"); + + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod4 (3)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod4 (3)"); + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod4 (4)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod4 (4)"); + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod4 (5)"); + testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod4 (5)"); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 10, skipped: 6); } @@ -208,6 +257,11 @@ public void TestMethod2() [TestClass] public class TestClassWithDataSourcesUsingIgnoreMessage { + private readonly TestContext _testContext; + + public TestClassWithDataSourcesUsingIgnoreMessage(TestContext testContext) + => _testContext = testContext; + [TestMethod] // 1 skipped, 2 pass [DataRow(0, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] [DataRow(1, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold, IgnoreMessage = "This data row is ignored")] @@ -217,8 +271,8 @@ public void TestMethod1(int i) } [TestMethod] // 1 skipped (folded), 3 pass - [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] - [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold, IgnoreMessage = "This source is ignored")] + [DynamicData("Data1", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold)] + [DynamicData("Data2", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Fold, IgnoreMessage = "This source is ignored")] public void TestMethod2(int i) { } @@ -232,13 +286,25 @@ public void TestMethod3(int i) } [TestMethod] // 3 skipped (unfolded), 3 pass - [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold)] - [DynamicData("Data", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold, IgnoreMessage = "This source is ignored")] + [DynamicData("Data1", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold)] + [DynamicData("Data2", UnfoldingStrategy = TestDataSourceUnfoldingStrategy.Unfold, IgnoreMessage = "This source is ignored")] public void TestMethod4(int i) { } - public static IEnumerable Data + [TestInitialize] + public void TestInit() + { + Console.WriteLine($"TestInitialize: {_testContext.TestDisplayName}"); + } + + [TestCleanup] + public void TestClean() + { + Console.WriteLine($"TestCleanup: {_testContext.TestDisplayName}"); + } + + public static IEnumerable Data1 { get { @@ -247,6 +313,16 @@ public static IEnumerable Data yield return new object[] { 2 }; } } + + public static IEnumerable Data2 + { + get + { + yield return new object[] { 3 }; + yield return new object[] { 4 }; + yield return new object[] { 5 }; + } + } } """; } From 95fbe376e7c860a5d88dacf5c0f9a33e2a2af92a Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 2 Jan 2025 16:17:57 +0100 Subject: [PATCH 11/11] Adjust test --- .../IgnoreTests.cs | 82 +++++++++++++------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs index 3d3886f171..519f74159c 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/IgnoreTests.cs @@ -66,35 +66,63 @@ public async Task WhenSpecificDataSourceIsIgnoredViaIgnoreMessageProperty() // Assert testHostResult.AssertExitCodeIs(ExitCodes.Success); - testHostResult.AssertOutputMatchesRegex(""" - TestInitialize: TestMethod1 \(0\) - TestCleanup: TestMethod1 \(0\) - TestInitialize: TestMethod1 \(2\) - TestCleanup: TestMethod1 \(2\) - TestInitialize: TestMethod2 \(0\) - TestCleanup: TestMethod2 \(0\) - TestInitialize: TestMethod2 \(1\) - TestCleanup: TestMethod2 \(1\) - TestInitialize: TestMethod2 \(2\) - TestCleanup: TestMethod2 \(2\) - skipped TestMethod1 \(\d+ms\) - skipped TestMethod2 \(\d+ms\) - TestInitialize: TestMethod3 \(0\) - TestCleanup: TestMethod3 \(0\) - skipped TestMethod3 \(1\) \(\d+ms\) - TestInitialize: TestMethod3 \(2\) - TestCleanup: TestMethod3 \(2\) - TestInitialize: TestMethod4 \(0\) - TestCleanup: TestMethod4 \(0\) - TestInitialize: TestMethod4 \(1\) - TestCleanup: TestMethod4 \(1\) - TestInitialize: TestMethod4 \(2\) - TestCleanup: TestMethod4 \(2\) - skipped TestMethod4 \(3\) \(\d+ms\) - skipped TestMethod4 \(4\) \(\d+ms\) - skipped TestMethod4 \(5\) \(\d+ms\) + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod1 (0) + TestCleanup: TestMethod1 (0) """); + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod1 (2) + TestCleanup: TestMethod1 (2) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod2 (0) + TestCleanup: TestMethod2 (0) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod2 (1) + TestCleanup: TestMethod2 (1) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod2 (2) + TestCleanup: TestMethod2 (2) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod3 (0) + TestCleanup: TestMethod3 (0) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod3 (2) + TestCleanup: TestMethod3 (2) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod4 (0) + TestCleanup: TestMethod4 (0) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod4 (1) + TestCleanup: TestMethod4 (1) + """); + + testHostResult.AssertOutputContains(""" + TestInitialize: TestMethod4 (2) + TestCleanup: TestMethod4 (2) + """); + + testHostResult.AssertOutputContains("skipped TestMethod1"); + testHostResult.AssertOutputContains("skipped TestMethod2"); + testHostResult.AssertOutputContains("skipped TestMethod3 (1)"); + testHostResult.AssertOutputContains("skipped TestMethod4 (3)"); + testHostResult.AssertOutputContains("skipped TestMethod4 (4)"); + testHostResult.AssertOutputContains("skipped TestMethod4 (5)"); + testHostResult.AssertOutputDoesNotContain("TestInitialize: TestMethod1 (1)"); testHostResult.AssertOutputDoesNotContain("TestCleanup: TestMethod1 (1)");