diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 884fe813fb..dd62c99a75 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -86,10 +86,11 @@ internal ICollection EnumerateAssembly( DataRowAttribute.TestIdGenerationStrategy = testIdGenerationStrategy; DynamicDataAttribute.TestIdGenerationStrategy = testIdGenerationStrategy; - TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = ReflectHelper.GetTestDataSourceOptions(assembly)?.UnfoldingStrategy switch + TestDataSourceOptionsAttribute? testDataSourceOptionsAttribute = ReflectHelper.GetTestDataSourceOptions(assembly); + TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = testDataSourceOptionsAttribute?.UnfoldingStrategy switch { // When strategy is auto we want to unfold - TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.Unfold, + TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer, // When strategy is set, let's use it { } value => value, // When the attribute is not set, let's look at the legacy attribute @@ -103,7 +104,7 @@ internal ICollection EnumerateAssembly( // as the ID generator will ignore it. (null, TestIdGenerationStrategy.Legacy) => TestDataSourceUnfoldingStrategy.Fold, #pragma warning restore CS0618 // Type or member is obsolete - _ => TestDataSourceUnfoldingStrategy.Unfold, + _ => TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer, }, }; @@ -115,8 +116,7 @@ internal ICollection EnumerateAssembly( continue; } - List testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, - dataSourcesUnfoldingStrategy, testIdGenerationStrategy, fixturesTests); + List testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, dataSourcesUnfoldingStrategy, testIdGenerationStrategy, fixturesTests); tests.AddRange(testsInType); } @@ -368,10 +368,12 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn try { bool isDataDriven = false; + int dataSourceIndex = -1; foreach (ITestDataSource dataSource in testDataSources) { + dataSourceIndex++; isDataDriven = true; - if (!TryUnfoldITestDataSource(dataSource, dataSourcesUnfoldingStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests)) + if (!TryUnfoldITestDataSource(dataSource, dataSourceIndex, dataSourcesUnfoldingStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests)) { // TODO: Improve multi-source design! // Ideally we would want to consider each data source separately but when one source cannot be expanded, @@ -401,7 +403,7 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn } } - private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List tests) + private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, int dataSourceIndex, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List tests) { var unfoldingCapability = dataSource as ITestDataSourceUnfoldingCapability; @@ -454,6 +456,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat // If strategy is DisplayName and we have a duplicate test name don't expand the test, bail out. #pragma warning disable CS0618 // Type or member is obsolete if (test.TestMethod.TestIdGenerationStrategy == TestIdGenerationStrategy.DisplayName +#pragma warning restore CS0618 // Type or member is obsolete && testDisplayNameFirstSeen.TryGetValue(discoveredTest.DisplayName!, out int firstIndexSeen)) { string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_DuplicateDisplayName, firstIndexSeen, index, discoveredTest.DisplayName); @@ -463,23 +466,36 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat // Duplicated display name so bail out. Caller will handle adding the original test. return false; } -#pragma warning restore CS0618 // Type or member is obsolete - try + if (unfoldingCapability?.UnfoldingStrategy == TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex + || (unfoldingCapability?.UnfoldingStrategy is null or TestDataSourceUnfoldingStrategy.Auto && dataSourcesUnfoldingStrategy == TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)) { - discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); + discoveredTest.TestMethod.SerializedData = new string[3] + { + TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex.ToString(), + dataSourceIndex.ToString(CultureInfo.InvariantCulture), + index.ToString(CultureInfo.InvariantCulture), + }; discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; } - catch (SerializationException ex) + else { - string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName); - warning += Environment.NewLine; - warning += ex.ToString(); - warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); + try + { + discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); + discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; + } + catch (SerializationException ex) + { + string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName); + warning += Environment.NewLine; + warning += ex.ToString(); + warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); - // Serialization failed for the type, bail out. Caller will handle adding the original test. - return false; + // Serialization failed for the type, bail out. Caller will handle adding the original test. + return false; + } } discoveredTests.Add(discoveredTest); diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index 562bb7376e..96c7223482 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -181,17 +181,12 @@ internal UnitTestResult[] RunTestMethod() bool isDataDriven = false; var parentStopwatch = Stopwatch.StartNew(); - if (_test.DataType == DynamicDataType.ITestDataSource) + if (TryExecuteITestDataSource(results)) { - object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData); - TestResult[] testResults = ExecuteTestWithDataSource(null, data); - results.AddRange(testResults); - } - else if (TryExecuteDataSourceBasedTests(results)) - { - isDataDriven = true; + // Do nothing } - else if (TryExecuteFoldedDataDrivenTests(results)) + else if (TryExecuteDataSourceBasedTests(results) + || TryExecuteFoldedDataDrivenTests(results)) { isDataDriven = true; } @@ -254,6 +249,44 @@ internal UnitTestResult[] RunTestMethod() return results.ToUnitTestResults(); } + private bool TryExecuteITestDataSource(List results) + { + if (_test.DataType != DynamicDataType.ITestDataSource) + { + return false; + } + + UTF.ITestDataSource? dataSource; + object?[]? data; + if (_test.SerializedData?.Length == 3) + { + if (!Enum.TryParse(_test.SerializedData[0], out TestDataSourceUnfoldingStrategy _) + || !int.TryParse(_test.SerializedData[1], out int dataSourceIndex) + || !int.TryParse(_test.SerializedData[2], out int dataIndex)) + { + throw ApplicationStateGuard.Unreachable(); + } + + dataSource = _testMethodInfo.GetAttributes(false)?.OfType().Skip(dataSourceIndex).FirstOrDefault(); + if (dataSource is null) + { + throw ApplicationStateGuard.Unreachable(); + } + + data = dataSource.GetData(_testMethodInfo.MethodInfo).Skip(dataIndex).FirstOrDefault(); + } + else + { + dataSource = null; + data = DataSerializationHelper.Deserialize(_test.SerializedData); + } + + TestResult[] testResults = ExecuteTestWithDataSource(dataSource, data); + results.AddRange(testResults); + + return true; + } + private bool TryExecuteDataSourceBasedTests(List results) { DataSourceAttribute[] dataSourceAttribute = _testMethodInfo.GetAttributes(false); diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs index 2f5ced8910..41b542b45c 100644 --- a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs +++ b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs @@ -22,17 +22,34 @@ public enum TestDataSourceUnfoldingStrategy : byte /// /// MSTest will decide whether to unfold the parameterized test based on value from the assembly level attribute /// . If no assembly level attribute is specified, then the default - /// configuration is to unfold. + /// configuration is to unfold using . /// Auto, /// /// Each data row is treated as a separate test case. /// + [Obsolete("Use 'UnfoldUsingDataContractJsonSerializer' instead")] Unfold, /// /// The parameterized test is not unfolded; all data rows are treated as a single test case. /// Fold, + + /// + /// Each data row is treated as a separate test case, and the data is unfolded using + /// . + /// + UnfoldUsingDataContractJsonSerializer, + + /// + /// Each data row is treated as a separate test case, and the data is unfolded using the data + /// source index and data index. + /// + /// + /// Using this strategy will alter the test ID if the data source is reordered, as it depends + /// on the index of the data. This may affect the ability to track test cases over time. + /// + UnfoldUsingDataIndex, } diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index 3ceaf8e88e..bd2ef498a5 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -6,6 +6,8 @@ Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAtt Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer = 3 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex = 4 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Throws(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsAsync(System.Func! action, string! message = "", params object![]! messageArgs) -> System.Threading.Tasks.Task! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactly(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs index 4be71e0050..142afffc45 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Collections.Immutable; + using Microsoft.MSTestV2.CLIAutomation; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; namespace MSTest.IntegrationTests; @@ -15,8 +18,8 @@ public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRow string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -34,8 +37,8 @@ public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_Simp string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -50,8 +53,8 @@ public void DataRowsExecuteWithRequiredAndOptionalParameters() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -67,8 +70,8 @@ public void DataRowsExecuteWithParamsArrayParameter() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -85,8 +88,8 @@ public void DataRowsFailWhenInvalidArgumentsProvided() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsFailed( @@ -102,8 +105,8 @@ public void DataRowsShouldSerializeDoublesProperly() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -118,8 +121,8 @@ public void DataRowsShouldSerializeMixedTypesProperly() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -133,8 +136,8 @@ public void DataRowsShouldSerializeEnumsProperly() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -151,8 +154,8 @@ public void DataRowsShouldHandleNonSerializableValues() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsDiscovered( @@ -172,8 +175,8 @@ public void ExecuteDataRowTests_Enums() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -247,8 +250,8 @@ public void ExecuteDataRowTests_NonSerializablePaths() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -265,8 +268,8 @@ public void ExecuteDataRowTests_Regular() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -326,8 +329,8 @@ public void GetDisplayName_AfterOverriding_GetsTheNewDisplayName() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName"); + ImmutableArray testResults = RunTests(testCases); VerifyE2E.TestsPassed( testResults, @@ -340,12 +343,30 @@ public void ParameterizedTestsWithTestMethodSettingDisplayName_DataIsPrefixWithD string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest"); + ImmutableArray testResults = RunTests(testCases); VerifyE2E.TestsPassed( testResults, "SomeCustomDisplayName2 (\"SomeData\")", "SomeCustomDisplayName3 (\"SomeData\")"); } + + public void RunTestsFailingWhenUsingSerialization() + { + // Arrange + string assemblyPath = GetAssetFullPath(TestAssetName); + + // Act + ImmutableArray testCases = DiscoverTests(assemblyPath, "ClassName~DataRowTests_Index"); + ImmutableArray testResults = RunTests(testCases); + + // Assert + VerifyE2E.TestsPassed( + testResults, + "NestedInputTypesInvalid (0,System.Object[])", + "NestedInputTypes (0,System.Object[])", + "NestedInputTypes (0,System.Object[])", + "NestedInputTypes (0,System.Object[])"); + } } diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs index 44dc75da83..d5c01f2636 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Immutable; @@ -116,4 +116,51 @@ public void ExecuteNonExpandableDynamicDataTests() "TestMethodSourceOnCurrentType (2,b)"); VerifyE2E.FailedTestCount(testResults, 0); } + + public void ExecuteTestsFailingWhenUsingSerialization() + { + // Arrange + string assemblyPath = GetAssetFullPath(TestAssetName); + + // Act + ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DynamicDataTests_Index"); + ImmutableArray testResults = RunTests(testCases); + + // Assert + VerifyE2E.TestsPassed( + testResults, + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "TestReadonlyCollectionData (,DataRowTestProject.DynamicDataTests_Index+MyData)", + "TestUnlimitedNatural (0,*,False)", + "TestUnlimitedNatural (0,0,True)", + "ValidateExMessage (DataRowTestProject.DynamicDataTests_Index+InvalidUpdateException: Test exception message)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (0,Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-0.45,Vector2D: -17.433 / -8.196,Vector2D: 7.845 / 3.688)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-0.73,Vector2D: 1.000 / 2.000,Vector2D: -0.730 / -1.460)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (1023.56,Vector2D: 12.430 / -2.754,Vector2D: 12722.851 / -2818.884)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-13.25,Vector2D: 4.230 / 6.812,Vector2D: -56.048 / -90.257)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (2.1,Vector2D: 2.000 / 3.000,Vector2D: 4.200 / 6.300)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (22.415,Vector2D: -3.400 / 2.750,Vector2D: -76.211 / 61.641)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-3.5,Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 0.000 / 0.000,0,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 0.000 / 0.000,-3.5,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 1.000 / 2.000,-0.73,Vector2D: -0.730 / -1.460)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 12.430 / -2.754,1023.56,Vector2D: 12722.851 / -2818.884)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: -17.433 / -8.196,-0.45,Vector2D: 7.845 / 3.688)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 2.000 / 3.000,2.1,Vector2D: 4.200 / 6.300)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: -3.400 / 2.750,22.415,Vector2D: -76.211 / 61.641)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 4.230 / 6.812,-13.25,Vector2D: -56.048 / -90.257)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000,0)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 0.000 / 0.000,Vector2D: 2.000 / 3.000,0)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -0.150 / 2.030,Vector2D: 4.230 / 6.812,13.1935961)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 1.000 / 2.000,Vector2D: 0.000 / 0.000,0)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -1.000 / -2.000,Vector2D: -3.400 / 2.750,-2.1)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 1.000 / 3.000,Vector2D: 1.000 / 2.000,7)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -22.723 / -78.298,Vector2D: -17.433 / -8.196,1037.82079593)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 3.355 / -2.211,Vector2D: 12.430 / -2.754,47.791744)"); + VerifyE2E.FailedTestCount(testResults, 0); + } } diff --git a/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs b/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs new file mode 100644 index 0000000000..cf0b206191 --- /dev/null +++ b/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DataRowTestProject; + +[TestClass] +public class DataRowTests_Index +{ + #region // https://github.com/microsoft/testfx/issues/2390 + + [TestMethod] + [DataRow((byte)0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + [DataRow((short)0, new object[] { (short)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + [DataRow(0L, new object[] { 0L }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void NestedInputTypes(object org, object nested) + => Assert.IsTrue( + org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), + string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name)); + + [TestMethod] + [DataRow(0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void NestedInputTypesInvalid(object org, object nested) + => Assert.IsFalse( + org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), + string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name)); + + #endregion +} diff --git a/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs new file mode 100644 index 0000000000..69cf9c8a07 --- /dev/null +++ b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.ObjectModel; +using System.Runtime.Serialization; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DataRowTestProject; + +[TestClass] +public class DynamicDataTests_Index +{ + #region https://github.com/microsoft/testfx/issues/908 + + [TestMethod] + [DynamicData(nameof(AddTestCases), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void Add_ShouldAddTheExpectedValues(Collection systemUnderTest, IEnumerable itemsToAdd, Collection expected) + { + // The actual tested method is irrelevant. Executing this empty tests provokes the error + } + + public static IEnumerable AddTestCases + { + get + { + var sut = new Collection(); + var expected = new Collection(); + yield return new object[] { sut, Enumerable.Empty(), expected }; + + sut = new Collection() { "1" }; + expected = new Collection() { "1" }; + yield return new object[] { sut, Enumerable.Empty(), expected }; + + sut = new Collection(); + expected = new Collection() { "1" }; + yield return new object[] { sut, new[] { "1" }, expected }; + + sut = new Collection() { "1", "a", "b" }; + expected = new Collection() { "1", "a", "b", "z", "j" }; + yield return new object[] { sut, new[] { "z", "j" }, expected }; + } + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1022 + + [TestMethod] + [DynamicData(nameof(UnlimitedNaturalData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void TestUnlimitedNatural(UnlimitedNatural testObject, UnlimitedNatural other, bool expected) + { + bool actual = testObject.Equals(other); + Assert.AreEqual(expected, actual); + } + + public static IEnumerable UnlimitedNaturalData + { + get + { + yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Infinite, false }; + yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Zero, true }; + } + } + +#pragma warning disable CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode() +#pragma warning disable CA2231 // Overload operator equals on overriding value type Equals + public readonly struct UnlimitedNatural : IEquatable +#pragma warning restore CA2231 // Overload operator equals on overriding value type Equals +#pragma warning restore CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode() + { + public static readonly UnlimitedNatural Infinite; + public static readonly UnlimitedNatural One = new(1); + public static readonly UnlimitedNatural Zero = new(0); + + public UnlimitedNatural(uint? value) + => Value = value; + + public uint? Value { get; } + + public override string ToString() + => Value?.ToString(CultureInfo.InvariantCulture) ?? "*"; + + public override bool Equals(object obj) + => obj is UnlimitedNatural other && Equals(other); + + public bool Equals(UnlimitedNatural other) + => Value == other.Value; + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1037 + + [DynamicData(nameof(TestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + [TestMethod] + public void ValidateExMessage(Exception ex) + => Assert.AreEqual(ex?.Message, "Test exception message"); + + private static IEnumerable TestData() + => new List() + { + new object[] { new InvalidUpdateException("Test exception message") }, + }; + + [Serializable] + public class InvalidUpdateException : Exception + { + public InvalidUpdateException(string message) + : base(message) + { + } + + public InvalidUpdateException() + { + } + + [ExcludeFromCodeCoverage] + protected InvalidUpdateException(SerializationInfo serializationInfo, StreamingContext streamingContext) + { + } + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1094 + + [TestMethod] + [DynamicData(nameof(Create_test_cases_for_multiplication_of_vector_and_real), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector(Vector2D v, double d, Vector2D expected) + { + Vector2D actual = v * d; + double delta = 0.00001; + + Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly."); + Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly."); + } + + [TestMethod] + [DynamicData(nameof(Create_test_cases_for_multiplication_of_real_and_vector), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector(double d, Vector2D v, Vector2D expected) + { + Vector2D actual = d * v; + double delta = 0.00001; + + Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly."); + Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly."); + } + + [TestMethod] + [DynamicData(nameof(Create_test_cases_for_scalar_product_of_two_vectors), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product(Vector2D v, Vector2D w, double expected) + { + double actual = v * w; + double delta = 0.00001; + + Assert.AreEqual(expected, actual, delta, "The scalar product hasn't been computed correctly."); + } + + private static IEnumerable Create_test_cases_for_multiplication_of_vector_and_real() + => new[] + { + new object[] { new Vector2D(0.0, 0.0), 0.0, new Vector2D(0.0, 0.0) }, + new object[] { new Vector2D(0.0, 0.0), -3.5, new Vector2D(0.0, 0.0) }, + new object[] { new Vector2D(2.0, 3.0), 2.1, new Vector2D(4.2, 6.3) }, + new object[] { new Vector2D(1.0, 2.0), -0.73, new Vector2D(-0.73, -1.46) }, + new object[] { new Vector2D(-3.4, 2.75), 22.415, new Vector2D(-76.211, 61.64125) }, + new object[] { new Vector2D(12.43, -2.754), 1023.56, new Vector2D(12722.8508, -2818.88424) }, + new object[] { new Vector2D(4.23, 6.81187), -13.25, new Vector2D(-56.0475, -90.2572775) }, + new object[] { new Vector2D(-17.4327, -8.1956), -0.45, new Vector2D(7.844715, 3.68802) }, + }; + + private static IEnumerable Create_test_cases_for_multiplication_of_real_and_vector() + => new[] + { + new object[] { 0.0, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) }, + new object[] { -3.5, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) }, + new object[] { 2.1, new Vector2D(2.0, 3.0), new Vector2D(4.2, 6.3) }, + new object[] { -0.73, new Vector2D(1.0, 2.0), new Vector2D(-0.73, -1.46) }, + new object[] { 22.415, new Vector2D(-3.4, 2.75), new Vector2D(-76.211, 61.64125) }, + new object[] { 1023.56, new Vector2D(12.43, -2.754), new Vector2D(12722.8508, -2818.88424) }, + new object[] { -13.25, new Vector2D(4.23, 6.81187), new Vector2D(-56.0475, -90.2572775) }, + new object[] { -0.45, new Vector2D(-17.4327, -8.1956), new Vector2D(7.844715, 3.68802) }, + }; + + private static IEnumerable Create_test_cases_for_scalar_product_of_two_vectors() + => new[] + { + new object[] { new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0), 0.0 }, + new object[] { new Vector2D(1.0, 2.0), new Vector2D(0.0, 0.0), 0.0 }, + new object[] { new Vector2D(0.0, 0.0), new Vector2D(2.0, 3.0), 0.0 }, + new object[] { new Vector2D(1.0, 3.0), new Vector2D(1.0, 2.0), 7.0 }, + new object[] { new Vector2D(-1.0, -2.0), new Vector2D(-3.4, 2.75), -2.1 }, + new object[] { new Vector2D(3.355, -2.211), new Vector2D(12.43, -2.754), 47.791744 }, + new object[] { new Vector2D(-0.15, 2.03), new Vector2D(4.23, 6.81187), 13.1935961 }, + new object[] { new Vector2D(-22.7231, -78.2976), new Vector2D(-17.4327, -8.1956), 1037.82079593 }, + }; + + public readonly struct Vector2D + { + public Vector2D(double u, double v) + { + U = u; + V = v; + } + + public double U { get; } + + public double V { get; } + + public override string ToString() + => FormattableString.Invariant($"Vector2D: {U:F3} / {V:F3}"); + + public static Vector2D operator *(Vector2D v, double d) + => new(v.U * d, v.V * d); + + public static Vector2D operator *(double d, Vector2D v) + => new(d * v.U, d * v.V); + + public static double operator *(Vector2D v, Vector2D w) + => (v.U * w.U) + (v.V * w.V); + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1588 + + [TestMethod] + [DynamicData(nameof(GetTestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + public void TestReadonlyCollectionData(string someString, MyData foo) + { + } + + private static IEnumerable GetTestData() + { + yield return new object[] + { + string.Empty, new MyData(), + }; + } + + public class MyData + { + private readonly SortedSet _values; + + public MyData() => _values = new SortedSet(); + + public IEnumerable Values => _values; + } + + #endregion +}