From 4475789cb2ca4a378cac02f7b2779c9f61eadd64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 29 Dec 2024 14:32:01 +0100 Subject: [PATCH 1/6] Offer alternative to serialization of data --- .../Discovery/AssemblyEnumerator.cs | 62 +++++++++++++------ .../Execution/TestMethodRunner.cs | 53 +++++++++++++--- .../Attributes/DataSource/DataRowAttribute.cs | 5 +- .../DataSource/DynamicDataAttribute.cs | 5 +- .../TestDataSourceOptionsAttribute.cs | 35 ++++++++++- .../Interfaces/ITestDataIdentifierStrategy.cs | 40 ++++++++++++ .../PublicAPI/PublicAPI.Unshipped.txt | 13 ++++ 7 files changed, 183 insertions(+), 30 deletions(-) create mode 100644 src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 884fe813fb..4259309c2e 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -86,7 +86,8 @@ internal ICollection EnumerateAssembly( DataRowAttribute.TestIdGenerationStrategy = testIdGenerationStrategy; DynamicDataAttribute.TestIdGenerationStrategy = testIdGenerationStrategy; - TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = ReflectHelper.GetTestDataSourceOptions(assembly)?.UnfoldingStrategy switch + TestDataSourceOptionsAttribute? testDataSourceOptionsAttribute = ReflectHelper.GetTestDataSourceOptions(assembly); + TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = testDataSourceOptionsAttribute?.UnfoldingStrategy switch { // When strategy is auto we want to unfold TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.Unfold, @@ -107,6 +108,14 @@ internal ICollection EnumerateAssembly( }, }; + TestDataIdentifierStrategy dataSourcesIdentifierStrategy = testDataSourceOptionsAttribute?.IdentifierStrategy switch + { + // When strategy is null or auto we want to use the data contract serialization + null or TestDataIdentifierStrategy.Auto => TestDataIdentifierStrategy.DataContractSerialization, + // When strategy is set, let's use it + { } value => value, + }; + Dictionary? testRunParametersFromRunSettings = RunSettingsUtilities.GetTestRunParameters(runSettingsXml); foreach (Type type in types) { @@ -115,8 +124,7 @@ internal ICollection EnumerateAssembly( continue; } - List testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, - dataSourcesUnfoldingStrategy, testIdGenerationStrategy, fixturesTests); + List testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, dataSourcesUnfoldingStrategy, dataSourcesIdentifierStrategy, testIdGenerationStrategy, fixturesTests); tests.AddRange(testsInType); } @@ -216,6 +224,7 @@ private List DiscoverTestsInType( List warningMessages, bool discoverInternals, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, + TestDataIdentifierStrategy dataSourcesIdentifierStrategy, TestIdGenerationStrategy testIdGenerationStrategy, HashSet fixturesTests) { @@ -246,7 +255,7 @@ private List DiscoverTestsInType( AddFixtureTests(testMethodInfo, tests, fixturesTests); } - if (TryUnfoldITestDataSources(test, testMethodInfo, dataSourcesUnfoldingStrategy, tests)) + if (TryUnfoldITestDataSources(test, testMethodInfo, dataSourcesUnfoldingStrategy, dataSourcesIdentifierStrategy, tests)) { continue; } @@ -348,7 +357,7 @@ static UnitTestElement GetFixtureTest(string classFullName, string assemblyLocat } } - private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodInfo testMethodInfo, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, List tests) + private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodInfo testMethodInfo, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, TestDataIdentifierStrategy dataSourcesIdentifierStrategy, List tests) { // It should always be `true`, but if any part of the chain is obsolete; it might not contain those. // Since we depend on those properties, if they don't exist, we bail out early. @@ -368,10 +377,12 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn try { bool isDataDriven = false; + int dataSourceIndex = -1; foreach (ITestDataSource dataSource in testDataSources) { + dataSourceIndex++; isDataDriven = true; - if (!TryUnfoldITestDataSource(dataSource, dataSourcesUnfoldingStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests)) + if (!TryUnfoldITestDataSource(dataSource, dataSourceIndex, dataSourcesUnfoldingStrategy, dataSourcesIdentifierStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests)) { // TODO: Improve multi-source design! // Ideally we would want to consider each data source separately but when one source cannot be expanded, @@ -401,7 +412,7 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn } } - private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List tests) + private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, int dataSourceIndex, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, TestDataIdentifierStrategy dataSourcesIdentifierStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List tests) { var unfoldingCapability = dataSource as ITestDataSourceUnfoldingCapability; @@ -454,6 +465,7 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat // If strategy is DisplayName and we have a duplicate test name don't expand the test, bail out. #pragma warning disable CS0618 // Type or member is obsolete if (test.TestMethod.TestIdGenerationStrategy == TestIdGenerationStrategy.DisplayName +#pragma warning restore CS0618 // Type or member is obsolete && testDisplayNameFirstSeen.TryGetValue(discoveredTest.DisplayName!, out int firstIndexSeen)) { string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_DuplicateDisplayName, firstIndexSeen, index, discoveredTest.DisplayName); @@ -463,23 +475,37 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, TestDat // Duplicated display name so bail out. Caller will handle adding the original test. return false; } -#pragma warning restore CS0618 // Type or member is obsolete - try + TestDataIdentifierStrategy currentDataSourceIdentifierStrategy = (dataSource as ITestDataIdentifierStrategy)?.IdentifierStrategy ?? TestDataIdentifierStrategy.Auto; + if (currentDataSourceIdentifierStrategy == TestDataIdentifierStrategy.DataIndex + || (currentDataSourceIdentifierStrategy == TestDataIdentifierStrategy.Auto && dataSourcesIdentifierStrategy == TestDataIdentifierStrategy.DataIndex)) { - discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); + discoveredTest.TestMethod.SerializedData = new string[3] + { + TestDataIdentifierStrategy.DataIndex.ToString(), + dataSourceIndex.ToString(CultureInfo.InvariantCulture), + index.ToString(CultureInfo.InvariantCulture), + }; discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; } - catch (SerializationException ex) + else { - string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName); - warning += Environment.NewLine; - warning += ex.ToString(); - warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); + try + { + discoveredTest.TestMethod.SerializedData = DataSerializationHelper.Serialize(d); + discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; + } + catch (SerializationException ex) + { + string warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute_CannotSerialize, index, discoveredTest.DisplayName); + warning += Environment.NewLine; + warning += ex.ToString(); + warning = string.Format(CultureInfo.CurrentCulture, Resource.CannotExpandIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, warning); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"DynamicDataEnumerator: {warning}"); - // Serialization failed for the type, bail out. Caller will handle adding the original test. - return false; + // Serialization failed for the type, bail out. Caller will handle adding the original test. + return false; + } } discoveredTests.Add(discoveredTest); diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index 562bb7376e..920b77909c 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -181,17 +181,12 @@ internal UnitTestResult[] RunTestMethod() bool isDataDriven = false; var parentStopwatch = Stopwatch.StartNew(); - if (_test.DataType == DynamicDataType.ITestDataSource) + if (TryExecuteITestDataSource(results)) { - object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData); - TestResult[] testResults = ExecuteTestWithDataSource(null, data); - results.AddRange(testResults); - } - else if (TryExecuteDataSourceBasedTests(results)) - { - isDataDriven = true; + // Do nothing } - else if (TryExecuteFoldedDataDrivenTests(results)) + else if (TryExecuteDataSourceBasedTests(results) + || TryExecuteFoldedDataDrivenTests(results)) { isDataDriven = true; } @@ -254,6 +249,46 @@ internal UnitTestResult[] RunTestMethod() return results.ToUnitTestResults(); } + private bool TryExecuteITestDataSource(List results) + { + if (_test.DataType != DynamicDataType.ITestDataSource) + { + return false; + } + + UTF.ITestDataSource? dataSource; + object?[]? data; + if (_test.SerializedData?.Length == 3 + && Enum.TryParse(_test.SerializedData[0], out TestDataIdentifierStrategy _)) + { + if (!int.TryParse(_test.SerializedData[1], out int dataSourceIndex) + || !int.TryParse(_test.SerializedData[2], out int dataIndex)) + { + // TODO + throw new InvalidOperationException(); + } + + dataSource = _testMethodInfo.GetAttributes(false)?.OfType().Skip(dataSourceIndex).FirstOrDefault(); + if (dataSource is null) + { + // TODO + throw new InvalidOperationException(); + } + + data = dataSource.GetData(_testMethodInfo.MethodInfo).Skip(dataIndex).FirstOrDefault(); + } + else + { + dataSource = null; + data = DataSerializationHelper.Deserialize(_test.SerializedData); + } + + TestResult[] testResults = ExecuteTestWithDataSource(dataSource, data); + results.AddRange(testResults); + + return true; + } + private bool TryExecuteDataSourceBasedTests(List results) { DataSourceAttribute[] dataSourceAttribute = _testMethodInfo.GetAttributes(false); diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs index 4fdb946813..d0b018d796 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// Attribute to define in-line data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability +public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability, ITestDataIdentifierStrategy { /// /// Initializes a new instance of the class. @@ -56,6 +56,9 @@ public DataRowAttribute(string?[]? stringArrayData) /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; + /// + public TestDataIdentifierStrategy IdentifierStrategy { get; set; } = TestDataIdentifierStrategy.Auto; + /// public IEnumerable GetData(MethodInfo methodInfo) => [Data]; diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs index 48825c992c..53ec67ab5c 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs @@ -32,7 +32,7 @@ public enum DynamicDataSourceType /// Attribute to define dynamic data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability +public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability, ITestDataIdentifierStrategy { private readonly string _dynamicDataSourceName; private readonly DynamicDataSourceType _dynamicDataSourceType; @@ -114,6 +114,9 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; + /// + public TestDataIdentifierStrategy IdentifierStrategy { get; set; } = TestDataIdentifierStrategy.Auto; + /// public IEnumerable GetData(MethodInfo methodInfo) => DynamicDataProvider.Instance.GetData(_dynamicDataDeclaringType, _dynamicDataSourceType, _dynamicDataSourceName, methodInfo); diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs index 6550f28093..66d838ea75 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs @@ -18,10 +18,43 @@ public sealed class TestDataSourceOptionsAttribute : Attribute /// The to use when executing parameterized tests. /// public TestDataSourceOptionsAttribute(TestDataSourceUnfoldingStrategy unfoldingStrategy) - => UnfoldingStrategy = unfoldingStrategy; + : this(unfoldingStrategy, TestDataIdentifierStrategy.Auto) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The to use when executing parameterized tests. + /// + public TestDataSourceOptionsAttribute(TestDataIdentifierStrategy identifierStrategy) + : this(TestDataSourceUnfoldingStrategy.Auto, identifierStrategy) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The to use when executing parameterized tests. + /// + /// + /// The to use when executing parameterized tests. + /// + public TestDataSourceOptionsAttribute(TestDataSourceUnfoldingStrategy unfoldingStrategy, TestDataIdentifierStrategy identifierStrategy) + { + UnfoldingStrategy = unfoldingStrategy; + IdentifierStrategy = identifierStrategy; + } /// /// Gets the test unfolding strategy. /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; } + + /// + /// Gets the test data identifier strategy. + /// + public TestDataIdentifierStrategy IdentifierStrategy { get; } } diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs new file mode 100644 index 0000000000..9bfa00e945 --- /dev/null +++ b/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Defines the strategy for uniquely identifying test data. +/// This is only used when is set to . +/// +public interface ITestDataIdentifierStrategy +{ + /// + /// Gets the strategy used to uniquely identify test data. + /// + TestDataIdentifierStrategy IdentifierStrategy { get; } +} + +/// +/// Specifies the available strategies for identifying test data. +/// This is only relevant when is set to . +/// +public enum TestDataIdentifierStrategy : byte +{ + /// + /// Automatically determines the identifier strategy based on the assembly-level attribute + /// . If no attribute is specified, the default is + /// . + /// + Auto, + + /// + /// Identifies test data using data contract serialization. + /// + DataContractSerialization, + + /// + /// Identifies test data by its index in the data source. + /// + DataIndex, +} diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index 3ceaf8e88e..aced88d152 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -1,11 +1,24 @@ #nullable enable +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IdentifierStrategy.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.IdentifierStrategy.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataIdentifierStrategy.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy.Auto = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy.DataContractSerialization = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy.DataIndex = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy identifierStrategy) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy unfoldingStrategy, Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy identifierStrategy) -> void static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Throws(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsAsync(System.Func! action, string! message = "", params object![]! messageArgs) -> System.Threading.Tasks.Task! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactly(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! From d5fed9ba80e85154cb5ee3866a6ab50b9acd68b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 29 Dec 2024 16:03:42 +0100 Subject: [PATCH 2/6] Add tests --- .../Parameterized tests/DataRowTests.cs | 77 ++++-- .../Parameterized tests/DynamicDataTests.cs | 49 +++- .../DataRowTestProject/DataRowTests_Index.cs | 30 +++ .../DynamicDataTests_Index.cs | 252 ++++++++++++++++++ 4 files changed, 379 insertions(+), 29 deletions(-) create mode 100644 test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs create mode 100644 test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs index 4be71e0050..142afffc45 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DataRowTests.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Collections.Immutable; + using Microsoft.MSTestV2.CLIAutomation; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; namespace MSTest.IntegrationTests; @@ -15,8 +18,8 @@ public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerivedClassHasDataRow string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -34,8 +37,8 @@ public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_Simp string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -50,8 +53,8 @@ public void DataRowsExecuteWithRequiredAndOptionalParameters() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -67,8 +70,8 @@ public void DataRowsExecuteWithParamsArrayParameter() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -85,8 +88,8 @@ public void DataRowsFailWhenInvalidArgumentsProvided() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular&TestCategory~DataRowOptionalInvalidArguments"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsFailed( @@ -102,8 +105,8 @@ public void DataRowsShouldSerializeDoublesProperly() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular.DataRowTestDouble"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -118,8 +121,8 @@ public void DataRowsShouldSerializeMixedTypesProperly() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowTestMixed"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -133,8 +136,8 @@ public void DataRowsShouldSerializeEnumsProperly() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowEnums"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -151,8 +154,8 @@ public void DataRowsShouldHandleNonSerializableValues() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_DerivedClass.DataRowNonSerializable"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsDiscovered( @@ -172,8 +175,8 @@ public void ExecuteDataRowTests_Enums() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Enums"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -247,8 +250,8 @@ public void ExecuteDataRowTests_NonSerializablePaths() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_NonSerializablePaths"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -265,8 +268,8 @@ public void ExecuteDataRowTests_Regular() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DataRowTests_Regular"); + ImmutableArray testResults = RunTests(testCases); // Assert VerifyE2E.TestsPassed( @@ -326,8 +329,8 @@ public void GetDisplayName_AfterOverriding_GetsTheNewDisplayName() string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenGetDisplayName"); + ImmutableArray testResults = RunTests(testCases); VerifyE2E.TestsPassed( testResults, @@ -340,12 +343,30 @@ public void ParameterizedTestsWithTestMethodSettingDisplayName_DataIsPrefixWithD string assemblyPath = GetAssetFullPath(TestAssetName); // Act - System.Collections.Immutable.ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest"); - System.Collections.Immutable.ImmutableArray testResults = RunTests(testCases); + ImmutableArray testCases = DiscoverTests(assemblyPath, "TestCategory~OverriddenTestMethodDisplayNameForParameterizedTest"); + ImmutableArray testResults = RunTests(testCases); VerifyE2E.TestsPassed( testResults, "SomeCustomDisplayName2 (\"SomeData\")", "SomeCustomDisplayName3 (\"SomeData\")"); } + + public void RunTestsFailingWhenUsingSerialization() + { + // Arrange + string assemblyPath = GetAssetFullPath(TestAssetName); + + // Act + ImmutableArray testCases = DiscoverTests(assemblyPath, "ClassName~DataRowTests_Index"); + ImmutableArray testResults = RunTests(testCases); + + // Assert + VerifyE2E.TestsPassed( + testResults, + "NestedInputTypesInvalid (0,System.Object[])", + "NestedInputTypes (0,System.Object[])", + "NestedInputTypes (0,System.Object[])", + "NestedInputTypes (0,System.Object[])"); + } } diff --git a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs index 44dc75da83..d5c01f2636 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/Parameterized tests/DynamicDataTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Immutable; @@ -116,4 +116,51 @@ public void ExecuteNonExpandableDynamicDataTests() "TestMethodSourceOnCurrentType (2,b)"); VerifyE2E.FailedTestCount(testResults, 0); } + + public void ExecuteTestsFailingWhenUsingSerialization() + { + // Arrange + string assemblyPath = GetAssetFullPath(TestAssetName); + + // Act + ImmutableArray testCases = DiscoverTests(assemblyPath, testCaseFilter: "ClassName~DynamicDataTests_Index"); + ImmutableArray testResults = RunTests(testCases); + + // Assert + VerifyE2E.TestsPassed( + testResults, + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "Add_ShouldAddTheExpectedValues (System.Collections.ObjectModel.Collection`1[System.String],System.String[],System.Collections.ObjectModel.Collection`1[System.String])", + "TestReadonlyCollectionData (,DataRowTestProject.DynamicDataTests_Index+MyData)", + "TestUnlimitedNatural (0,*,False)", + "TestUnlimitedNatural (0,0,True)", + "ValidateExMessage (DataRowTestProject.DynamicDataTests_Index+InvalidUpdateException: Test exception message)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (0,Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-0.45,Vector2D: -17.433 / -8.196,Vector2D: 7.845 / 3.688)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-0.73,Vector2D: 1.000 / 2.000,Vector2D: -0.730 / -1.460)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (1023.56,Vector2D: 12.430 / -2.754,Vector2D: 12722.851 / -2818.884)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-13.25,Vector2D: 4.230 / 6.812,Vector2D: -56.048 / -90.257)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (2.1,Vector2D: 2.000 / 3.000,Vector2D: 4.200 / 6.300)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (22.415,Vector2D: -3.400 / 2.750,Vector2D: -76.211 / 61.641)", + "Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector (-3.5,Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 0.000 / 0.000,0,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 0.000 / 0.000,-3.5,Vector2D: 0.000 / 0.000)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 1.000 / 2.000,-0.73,Vector2D: -0.730 / -1.460)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 12.430 / -2.754,1023.56,Vector2D: 12722.851 / -2818.884)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: -17.433 / -8.196,-0.45,Vector2D: 7.845 / 3.688)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 2.000 / 3.000,2.1,Vector2D: 4.200 / 6.300)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: -3.400 / 2.750,22.415,Vector2D: -76.211 / 61.641)", + "Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector (Vector2D: 4.230 / 6.812,-13.25,Vector2D: -56.048 / -90.257)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 0.000 / 0.000,Vector2D: 0.000 / 0.000,0)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 0.000 / 0.000,Vector2D: 2.000 / 3.000,0)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -0.150 / 2.030,Vector2D: 4.230 / 6.812,13.1935961)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 1.000 / 2.000,Vector2D: 0.000 / 0.000,0)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -1.000 / -2.000,Vector2D: -3.400 / 2.750,-2.1)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 1.000 / 3.000,Vector2D: 1.000 / 2.000,7)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: -22.723 / -78.298,Vector2D: -17.433 / -8.196,1037.82079593)", + "Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product (Vector2D: 3.355 / -2.211,Vector2D: 12.430 / -2.754,47.791744)"); + VerifyE2E.FailedTestCount(testResults, 0); + } } diff --git a/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs b/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs new file mode 100644 index 0000000000..577935390b --- /dev/null +++ b/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DataRowTestProject; + +[TestClass] +public class DataRowTests_Index +{ + #region // https://github.com/microsoft/testfx/issues/2390 + + [TestMethod] + [DataRow((byte)0, new object[] { (byte)0 }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DataRow((short)0, new object[] { (short)0 }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DataRow(0L, new object[] { 0L }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void NestedInputTypes(object org, object nested) + => Assert.IsTrue( + org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), + string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name)); + + [TestMethod] + [DataRow(0, new object[] { (byte)0 }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void NestedInputTypesInvalid(object org, object nested) + => Assert.IsFalse( + org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), + string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name)); + + #endregion +} diff --git a/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs new file mode 100644 index 0000000000..b1c4532733 --- /dev/null +++ b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.ObjectModel; +using System.Runtime.Serialization; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DataRowTestProject; + +[TestClass] +public class DynamicDataTests_Index +{ + #region https://github.com/microsoft/testfx/issues/908 + + [TestMethod] + [DynamicData(nameof(AddTestCases), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void Add_ShouldAddTheExpectedValues(Collection systemUnderTest, IEnumerable itemsToAdd, Collection expected) + { + // The actual tested method is irrelevant. Executing this empty tests provokes the error + } + + public static IEnumerable AddTestCases + { + get + { + var sut = new Collection(); + var expected = new Collection(); + yield return new object[] { sut, Enumerable.Empty(), expected }; + + sut = new Collection() { "1" }; + expected = new Collection() { "1" }; + yield return new object[] { sut, Enumerable.Empty(), expected }; + + sut = new Collection(); + expected = new Collection() { "1" }; + yield return new object[] { sut, new[] { "1" }, expected }; + + sut = new Collection() { "1", "a", "b" }; + expected = new Collection() { "1", "a", "b", "z", "j" }; + yield return new object[] { sut, new[] { "z", "j" }, expected }; + } + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1022 + + [TestMethod] + [DynamicData(nameof(UnlimitedNaturalData), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void TestUnlimitedNatural(UnlimitedNatural testObject, UnlimitedNatural other, bool expected) + { + bool actual = testObject.Equals(other); + Assert.AreEqual(expected, actual); + } + + public static IEnumerable UnlimitedNaturalData + { + get + { + yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Infinite, false }; + yield return new object[] { UnlimitedNatural.Zero, UnlimitedNatural.Zero, true }; + } + } + +#pragma warning disable CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode() +#pragma warning disable CA2231 // Overload operator equals on overriding value type Equals + public readonly struct UnlimitedNatural : IEquatable +#pragma warning restore CA2231 // Overload operator equals on overriding value type Equals +#pragma warning restore CS0659 // Type overrides Object.Equals(object o) but does not override Object.GetHashCode() + { + public static readonly UnlimitedNatural Infinite; + public static readonly UnlimitedNatural One = new(1); + public static readonly UnlimitedNatural Zero = new(0); + + public UnlimitedNatural(uint? value) + => Value = value; + + public uint? Value { get; } + + public override string ToString() + => Value?.ToString(CultureInfo.InvariantCulture) ?? "*"; + + public override bool Equals(object obj) + => obj is UnlimitedNatural other && Equals(other); + + public bool Equals(UnlimitedNatural other) + => Value == other.Value; + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1037 + + [DynamicData(nameof(TestData), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [TestMethod] + public void ValidateExMessage(Exception ex) + => Assert.AreEqual(ex?.Message, "Test exception message"); + + private static IEnumerable TestData() + => new List() + { + new object[] { new InvalidUpdateException("Test exception message") }, + }; + + [Serializable] + public class InvalidUpdateException : Exception + { + public InvalidUpdateException(string message) + : base(message) + { + } + + public InvalidUpdateException() + { + } + + [ExcludeFromCodeCoverage] + protected InvalidUpdateException(SerializationInfo serializationInfo, StreamingContext streamingContext) + { + } + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1094 + + [TestMethod] + [DynamicData(nameof(Create_test_cases_for_multiplication_of_vector_and_real), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector(Vector2D v, double d, Vector2D expected) + { + Vector2D actual = v * d; + double delta = 0.00001; + + Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly."); + Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly."); + } + + [TestMethod] + [DynamicData(nameof(Create_test_cases_for_multiplication_of_real_and_vector), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector(double d, Vector2D v, Vector2D expected) + { + Vector2D actual = d * v; + double delta = 0.00001; + + Assert.AreEqual(expected.U, actual.U, delta, "The U-component of the vector hasn't been computed correctly."); + Assert.AreEqual(expected.V, actual.V, delta, "The V-component of the vector hasn't been computed correctly."); + } + + [TestMethod] + [DynamicData(nameof(Create_test_cases_for_scalar_product_of_two_vectors), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product(Vector2D v, Vector2D w, double expected) + { + double actual = v * w; + double delta = 0.00001; + + Assert.AreEqual(expected, actual, delta, "The scalar product hasn't been computed correctly."); + } + + private static IEnumerable Create_test_cases_for_multiplication_of_vector_and_real() + => new[] + { + new object[] { new Vector2D(0.0, 0.0), 0.0, new Vector2D(0.0, 0.0) }, + new object[] { new Vector2D(0.0, 0.0), -3.5, new Vector2D(0.0, 0.0) }, + new object[] { new Vector2D(2.0, 3.0), 2.1, new Vector2D(4.2, 6.3) }, + new object[] { new Vector2D(1.0, 2.0), -0.73, new Vector2D(-0.73, -1.46) }, + new object[] { new Vector2D(-3.4, 2.75), 22.415, new Vector2D(-76.211, 61.64125) }, + new object[] { new Vector2D(12.43, -2.754), 1023.56, new Vector2D(12722.8508, -2818.88424) }, + new object[] { new Vector2D(4.23, 6.81187), -13.25, new Vector2D(-56.0475, -90.2572775) }, + new object[] { new Vector2D(-17.4327, -8.1956), -0.45, new Vector2D(7.844715, 3.68802) }, + }; + + private static IEnumerable Create_test_cases_for_multiplication_of_real_and_vector() + => new[] + { + new object[] { 0.0, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) }, + new object[] { -3.5, new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0) }, + new object[] { 2.1, new Vector2D(2.0, 3.0), new Vector2D(4.2, 6.3) }, + new object[] { -0.73, new Vector2D(1.0, 2.0), new Vector2D(-0.73, -1.46) }, + new object[] { 22.415, new Vector2D(-3.4, 2.75), new Vector2D(-76.211, 61.64125) }, + new object[] { 1023.56, new Vector2D(12.43, -2.754), new Vector2D(12722.8508, -2818.88424) }, + new object[] { -13.25, new Vector2D(4.23, 6.81187), new Vector2D(-56.0475, -90.2572775) }, + new object[] { -0.45, new Vector2D(-17.4327, -8.1956), new Vector2D(7.844715, 3.68802) }, + }; + + private static IEnumerable Create_test_cases_for_scalar_product_of_two_vectors() + => new[] + { + new object[] { new Vector2D(0.0, 0.0), new Vector2D(0.0, 0.0), 0.0 }, + new object[] { new Vector2D(1.0, 2.0), new Vector2D(0.0, 0.0), 0.0 }, + new object[] { new Vector2D(0.0, 0.0), new Vector2D(2.0, 3.0), 0.0 }, + new object[] { new Vector2D(1.0, 3.0), new Vector2D(1.0, 2.0), 7.0 }, + new object[] { new Vector2D(-1.0, -2.0), new Vector2D(-3.4, 2.75), -2.1 }, + new object[] { new Vector2D(3.355, -2.211), new Vector2D(12.43, -2.754), 47.791744 }, + new object[] { new Vector2D(-0.15, 2.03), new Vector2D(4.23, 6.81187), 13.1935961 }, + new object[] { new Vector2D(-22.7231, -78.2976), new Vector2D(-17.4327, -8.1956), 1037.82079593 }, + }; + + public readonly struct Vector2D + { + public Vector2D(double u, double v) + { + U = u; + V = v; + } + + public double U { get; } + + public double V { get; } + + public override string ToString() + => FormattableString.Invariant($"Vector2D: {U:F3} / {V:F3}"); + + public static Vector2D operator *(Vector2D v, double d) + => new(v.U * d, v.V * d); + + public static Vector2D operator *(double d, Vector2D v) + => new(d * v.U, d * v.V); + + public static double operator *(Vector2D v, Vector2D w) + => (v.U * w.U) + (v.V * w.V); + } + + #endregion + + #region https://github.com/microsoft/testfx/issues/1588 + + [TestMethod] + [DynamicData(nameof(GetTestData), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + public void TestReadonlyCollectionData(string someString, MyData foo) + { + } + + private static IEnumerable GetTestData() + { + yield return new object[] + { + string.Empty, new MyData(), + }; + } + + public class MyData + { + private readonly SortedSet _values; + + public MyData() => _values = new SortedSet(); + + public IEnumerable Values => _values; + } + + #endregion +} From f01baa64719ceca61db23b2fb70d9e828e4b9be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 29 Dec 2024 16:03:53 +0100 Subject: [PATCH 3/6] Fix todos --- .../MSTest.TestAdapter/Execution/TestMethodRunner.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index 920b77909c..00338a6f73 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -258,21 +258,19 @@ private bool TryExecuteITestDataSource(List results) UTF.ITestDataSource? dataSource; object?[]? data; - if (_test.SerializedData?.Length == 3 - && Enum.TryParse(_test.SerializedData[0], out TestDataIdentifierStrategy _)) + if (_test.SerializedData?.Length == 3) { - if (!int.TryParse(_test.SerializedData[1], out int dataSourceIndex) + if (!Enum.TryParse(_test.SerializedData[0], out TestDataIdentifierStrategy _) + || !int.TryParse(_test.SerializedData[1], out int dataSourceIndex) || !int.TryParse(_test.SerializedData[2], out int dataIndex)) { - // TODO - throw new InvalidOperationException(); + throw ApplicationStateGuard.Unreachable(); } dataSource = _testMethodInfo.GetAttributes(false)?.OfType().Skip(dataSourceIndex).FirstOrDefault(); if (dataSource is null) { - // TODO - throw new InvalidOperationException(); + throw ApplicationStateGuard.Unreachable(); } data = dataSource.GetData(_testMethodInfo.MethodInfo).Skip(dataIndex).FirstOrDefault(); From f06e4d255632611ad26a08d4f988a375da464d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 29 Dec 2024 16:04:05 +0100 Subject: [PATCH 4/6] Add remarks about impact of reorder + test ID --- .../Interfaces/ITestDataIdentifierStrategy.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs index 9bfa00e945..4fb2616059 100644 --- a/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs +++ b/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs @@ -12,6 +12,10 @@ public interface ITestDataIdentifierStrategy /// /// Gets the strategy used to uniquely identify test data. /// + /// + /// Altering the strategy may change the test ID generated for each test case, potentially + /// causing issues if the test is being monitored or tracked over time. + /// TestDataIdentifierStrategy IdentifierStrategy { get; } } @@ -36,5 +40,9 @@ public enum TestDataIdentifierStrategy : byte /// /// Identifies test data by its index in the data source. /// + /// + /// Using this strategy will alter the test ID if the data source is reordered, as it depends + /// on the index of the data. This may affect the ability to track test cases over time. + /// DataIndex, } From ca492b31202497dc8b1b82421d59284a8d8b5a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 29 Dec 2024 16:19:37 +0100 Subject: [PATCH 5/6] Reuse the folding/unfolding capability --- .../Discovery/AssemblyEnumerator.cs | 30 ++++-------- .../Execution/TestMethodRunner.cs | 2 +- .../Attributes/DataSource/DataRowAttribute.cs | 5 +- .../DataSource/DynamicDataAttribute.cs | 5 +- .../TestDataSourceOptionsAttribute.cs | 35 +------------- .../Interfaces/ITestDataIdentifierStrategy.cs | 48 ------------------- .../ITestDataSourceUnfoldingCapability.cs | 19 +++++++- .../PublicAPI/PublicAPI.Unshipped.txt | 2 + .../DataRowTestProject/DataRowTests_Index.cs | 8 ++-- .../DynamicDataTests_Index.cs | 14 +++--- 10 files changed, 45 insertions(+), 123 deletions(-) delete mode 100644 src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs index 4259309c2e..dd62c99a75 100644 --- a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs @@ -90,7 +90,7 @@ internal ICollection EnumerateAssembly( TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy = testDataSourceOptionsAttribute?.UnfoldingStrategy switch { // When strategy is auto we want to unfold - TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.Unfold, + TestDataSourceUnfoldingStrategy.Auto => TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer, // When strategy is set, let's use it { } value => value, // When the attribute is not set, let's look at the legacy attribute @@ -104,18 +104,10 @@ internal ICollection EnumerateAssembly( // as the ID generator will ignore it. (null, TestIdGenerationStrategy.Legacy) => TestDataSourceUnfoldingStrategy.Fold, #pragma warning restore CS0618 // Type or member is obsolete - _ => TestDataSourceUnfoldingStrategy.Unfold, + _ => TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer, }, }; - TestDataIdentifierStrategy dataSourcesIdentifierStrategy = testDataSourceOptionsAttribute?.IdentifierStrategy switch - { - // When strategy is null or auto we want to use the data contract serialization - null or TestDataIdentifierStrategy.Auto => TestDataIdentifierStrategy.DataContractSerialization, - // When strategy is set, let's use it - { } value => value, - }; - Dictionary? testRunParametersFromRunSettings = RunSettingsUtilities.GetTestRunParameters(runSettingsXml); foreach (Type type in types) { @@ -124,7 +116,7 @@ internal ICollection EnumerateAssembly( continue; } - List testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, dataSourcesUnfoldingStrategy, dataSourcesIdentifierStrategy, testIdGenerationStrategy, fixturesTests); + List testsInType = DiscoverTestsInType(assemblyFileName, testRunParametersFromRunSettings, type, warnings, discoverInternals, dataSourcesUnfoldingStrategy, testIdGenerationStrategy, fixturesTests); tests.AddRange(testsInType); } @@ -224,7 +216,6 @@ private List DiscoverTestsInType( List warningMessages, bool discoverInternals, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, - TestDataIdentifierStrategy dataSourcesIdentifierStrategy, TestIdGenerationStrategy testIdGenerationStrategy, HashSet fixturesTests) { @@ -255,7 +246,7 @@ private List DiscoverTestsInType( AddFixtureTests(testMethodInfo, tests, fixturesTests); } - if (TryUnfoldITestDataSources(test, testMethodInfo, dataSourcesUnfoldingStrategy, dataSourcesIdentifierStrategy, tests)) + if (TryUnfoldITestDataSources(test, testMethodInfo, dataSourcesUnfoldingStrategy, tests)) { continue; } @@ -357,7 +348,7 @@ static UnitTestElement GetFixtureTest(string classFullName, string assemblyLocat } } - private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodInfo testMethodInfo, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, TestDataIdentifierStrategy dataSourcesIdentifierStrategy, List tests) + private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodInfo testMethodInfo, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, List tests) { // It should always be `true`, but if any part of the chain is obsolete; it might not contain those. // Since we depend on those properties, if they don't exist, we bail out early. @@ -382,7 +373,7 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn { dataSourceIndex++; isDataDriven = true; - if (!TryUnfoldITestDataSource(dataSource, dataSourceIndex, dataSourcesUnfoldingStrategy, dataSourcesIdentifierStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests)) + if (!TryUnfoldITestDataSource(dataSource, dataSourceIndex, dataSourcesUnfoldingStrategy, test, new(testMethodInfo.MethodInfo, test.DisplayName), tempListOfTests)) { // TODO: Improve multi-source design! // Ideally we would want to consider each data source separately but when one source cannot be expanded, @@ -412,7 +403,7 @@ private static bool TryUnfoldITestDataSources(UnitTestElement test, TestMethodIn } } - private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, int dataSourceIndex, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, TestDataIdentifierStrategy dataSourcesIdentifierStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List tests) + private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, int dataSourceIndex, TestDataSourceUnfoldingStrategy dataSourcesUnfoldingStrategy, UnitTestElement test, ReflectionTestMethodInfo methodInfo, List tests) { var unfoldingCapability = dataSource as ITestDataSourceUnfoldingCapability; @@ -476,13 +467,12 @@ private static bool TryUnfoldITestDataSource(ITestDataSource dataSource, int dat return false; } - TestDataIdentifierStrategy currentDataSourceIdentifierStrategy = (dataSource as ITestDataIdentifierStrategy)?.IdentifierStrategy ?? TestDataIdentifierStrategy.Auto; - if (currentDataSourceIdentifierStrategy == TestDataIdentifierStrategy.DataIndex - || (currentDataSourceIdentifierStrategy == TestDataIdentifierStrategy.Auto && dataSourcesIdentifierStrategy == TestDataIdentifierStrategy.DataIndex)) + if (unfoldingCapability?.UnfoldingStrategy == TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex + || (unfoldingCapability?.UnfoldingStrategy is null or TestDataSourceUnfoldingStrategy.Auto && dataSourcesUnfoldingStrategy == TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)) { discoveredTest.TestMethod.SerializedData = new string[3] { - TestDataIdentifierStrategy.DataIndex.ToString(), + TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex.ToString(), dataSourceIndex.ToString(CultureInfo.InvariantCulture), index.ToString(CultureInfo.InvariantCulture), }; diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs index 00338a6f73..96c7223482 100644 --- a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs @@ -260,7 +260,7 @@ private bool TryExecuteITestDataSource(List results) object?[]? data; if (_test.SerializedData?.Length == 3) { - if (!Enum.TryParse(_test.SerializedData[0], out TestDataIdentifierStrategy _) + if (!Enum.TryParse(_test.SerializedData[0], out TestDataSourceUnfoldingStrategy _) || !int.TryParse(_test.SerializedData[1], out int dataSourceIndex) || !int.TryParse(_test.SerializedData[2], out int dataIndex)) { diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs index d0b018d796..4fdb946813 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DataRowAttribute.cs @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting; /// Attribute to define in-line data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability, ITestDataIdentifierStrategy +public class DataRowAttribute : Attribute, ITestDataSource, ITestDataSourceUnfoldingCapability { /// /// Initializes a new instance of the class. @@ -56,9 +56,6 @@ public DataRowAttribute(string?[]? stringArrayData) /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; - /// - public TestDataIdentifierStrategy IdentifierStrategy { get; set; } = TestDataIdentifierStrategy.Auto; - /// public IEnumerable GetData(MethodInfo methodInfo) => [Data]; diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs index 53ec67ab5c..48825c992c 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs @@ -32,7 +32,7 @@ public enum DynamicDataSourceType /// Attribute to define dynamic data for a test method. /// [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] -public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability, ITestDataIdentifierStrategy +public sealed class DynamicDataAttribute : Attribute, ITestDataSource, ITestDataSourceEmptyDataSourceExceptionInfo, ITestDataSourceUnfoldingCapability { private readonly string _dynamicDataSourceName; private readonly DynamicDataSourceType _dynamicDataSourceType; @@ -114,9 +114,6 @@ public DynamicDataAttribute(string dynamicDataSourceName, Type dynamicDataDeclar /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; set; } = TestDataSourceUnfoldingStrategy.Auto; - /// - public TestDataIdentifierStrategy IdentifierStrategy { get; set; } = TestDataIdentifierStrategy.Auto; - /// public IEnumerable GetData(MethodInfo methodInfo) => DynamicDataProvider.Instance.GetData(_dynamicDataDeclaringType, _dynamicDataSourceType, _dynamicDataSourceName, methodInfo); diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs index 66d838ea75..6550f28093 100644 --- a/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/DataSource/TestDataSourceOptionsAttribute.cs @@ -18,43 +18,10 @@ public sealed class TestDataSourceOptionsAttribute : Attribute /// The to use when executing parameterized tests. /// public TestDataSourceOptionsAttribute(TestDataSourceUnfoldingStrategy unfoldingStrategy) - : this(unfoldingStrategy, TestDataIdentifierStrategy.Auto) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// - /// The to use when executing parameterized tests. - /// - public TestDataSourceOptionsAttribute(TestDataIdentifierStrategy identifierStrategy) - : this(TestDataSourceUnfoldingStrategy.Auto, identifierStrategy) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// - /// The to use when executing parameterized tests. - /// - /// - /// The to use when executing parameterized tests. - /// - public TestDataSourceOptionsAttribute(TestDataSourceUnfoldingStrategy unfoldingStrategy, TestDataIdentifierStrategy identifierStrategy) - { - UnfoldingStrategy = unfoldingStrategy; - IdentifierStrategy = identifierStrategy; - } + => UnfoldingStrategy = unfoldingStrategy; /// /// Gets the test unfolding strategy. /// public TestDataSourceUnfoldingStrategy UnfoldingStrategy { get; } - - /// - /// Gets the test data identifier strategy. - /// - public TestDataIdentifierStrategy IdentifierStrategy { get; } } diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs deleted file mode 100644 index 4fb2616059..0000000000 --- a/src/TestFramework/TestFramework/Interfaces/ITestDataIdentifierStrategy.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestTools.UnitTesting; - -/// -/// Defines the strategy for uniquely identifying test data. -/// This is only used when is set to . -/// -public interface ITestDataIdentifierStrategy -{ - /// - /// Gets the strategy used to uniquely identify test data. - /// - /// - /// Altering the strategy may change the test ID generated for each test case, potentially - /// causing issues if the test is being monitored or tracked over time. - /// - TestDataIdentifierStrategy IdentifierStrategy { get; } -} - -/// -/// Specifies the available strategies for identifying test data. -/// This is only relevant when is set to . -/// -public enum TestDataIdentifierStrategy : byte -{ - /// - /// Automatically determines the identifier strategy based on the assembly-level attribute - /// . If no attribute is specified, the default is - /// . - /// - Auto, - - /// - /// Identifies test data using data contract serialization. - /// - DataContractSerialization, - - /// - /// Identifies test data by its index in the data source. - /// - /// - /// Using this strategy will alter the test ID if the data source is reordered, as it depends - /// on the index of the data. This may affect the ability to track test cases over time. - /// - DataIndex, -} diff --git a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs index 2f5ced8910..41b542b45c 100644 --- a/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs +++ b/src/TestFramework/TestFramework/Interfaces/ITestDataSourceUnfoldingCapability.cs @@ -22,17 +22,34 @@ public enum TestDataSourceUnfoldingStrategy : byte /// /// MSTest will decide whether to unfold the parameterized test based on value from the assembly level attribute /// . If no assembly level attribute is specified, then the default - /// configuration is to unfold. + /// configuration is to unfold using . /// Auto, /// /// Each data row is treated as a separate test case. /// + [Obsolete("Use 'UnfoldUsingDataContractJsonSerializer' instead")] Unfold, /// /// The parameterized test is not unfolded; all data rows are treated as a single test case. /// Fold, + + /// + /// Each data row is treated as a separate test case, and the data is unfolded using + /// . + /// + UnfoldUsingDataContractJsonSerializer, + + /// + /// Each data row is treated as a separate test case, and the data is unfolded using the data + /// source index and data index. + /// + /// + /// Using this strategy will alter the test ID if the data source is reordered, as it depends + /// on the index of the data. This may affect the ability to track test cases over time. + /// + UnfoldUsingDataIndex, } diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index aced88d152..91031aad64 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -19,6 +19,8 @@ Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy.DataInde Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy identifierStrategy) -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy unfoldingStrategy, Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy identifierStrategy) -> void +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer = 3 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy +Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex = 4 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Throws(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsAsync(System.Func! action, string! message = "", params object![]! messageArgs) -> System.Threading.Tasks.Task! static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowsExactly(System.Action! action, string! message = "", params object![]! messageArgs) -> TException! diff --git a/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs b/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs index 577935390b..cf0b206191 100644 --- a/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs +++ b/test/IntegrationTests/TestAssets/DataRowTestProject/DataRowTests_Index.cs @@ -11,16 +11,16 @@ public class DataRowTests_Index #region // https://github.com/microsoft/testfx/issues/2390 [TestMethod] - [DataRow((byte)0, new object[] { (byte)0 }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] - [DataRow((short)0, new object[] { (short)0 }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] - [DataRow(0L, new object[] { 0L }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DataRow((byte)0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + [DataRow((short)0, new object[] { (short)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] + [DataRow(0L, new object[] { 0L }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void NestedInputTypes(object org, object nested) => Assert.IsTrue( org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), string.Concat("Expected ", org.GetType().Name, " but got ", ((object[])nested)[0].GetType().Name)); [TestMethod] - [DataRow(0, new object[] { (byte)0 }, IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DataRow(0, new object[] { (byte)0 }, UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void NestedInputTypesInvalid(object org, object nested) => Assert.IsFalse( org.GetType().Name.Equals(((object[])nested)[0].GetType().Name, StringComparison.Ordinal), diff --git a/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs index b1c4532733..69cf9c8a07 100644 --- a/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs +++ b/test/IntegrationTests/TestAssets/DynamicDataTestProject/DynamicDataTests_Index.cs @@ -14,7 +14,7 @@ public class DynamicDataTests_Index #region https://github.com/microsoft/testfx/issues/908 [TestMethod] - [DynamicData(nameof(AddTestCases), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DynamicData(nameof(AddTestCases), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void Add_ShouldAddTheExpectedValues(Collection systemUnderTest, IEnumerable itemsToAdd, Collection expected) { // The actual tested method is irrelevant. Executing this empty tests provokes the error @@ -47,7 +47,7 @@ public static IEnumerable AddTestCases #region https://github.com/microsoft/testfx/issues/1022 [TestMethod] - [DynamicData(nameof(UnlimitedNaturalData), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DynamicData(nameof(UnlimitedNaturalData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void TestUnlimitedNatural(UnlimitedNatural testObject, UnlimitedNatural other, bool expected) { bool actual = testObject.Equals(other); @@ -92,7 +92,7 @@ public bool Equals(UnlimitedNatural other) #region https://github.com/microsoft/testfx/issues/1037 - [DynamicData(nameof(TestData), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DynamicData(nameof(TestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] [TestMethod] public void ValidateExMessage(Exception ex) => Assert.AreEqual(ex?.Message, "Test exception message"); @@ -126,7 +126,7 @@ protected InvalidUpdateException(SerializationInfo serializationInfo, StreamingC #region https://github.com/microsoft/testfx/issues/1094 [TestMethod] - [DynamicData(nameof(Create_test_cases_for_multiplication_of_vector_and_real), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DynamicData(nameof(Create_test_cases_for_multiplication_of_vector_and_real), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vector(Vector2D v, double d, Vector2D expected) { Vector2D actual = v * d; @@ -137,7 +137,7 @@ public void Vector2D_op_Multiplication_Vector2D_double___valid_args___scaled_vec } [TestMethod] - [DynamicData(nameof(Create_test_cases_for_multiplication_of_real_and_vector), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DynamicData(nameof(Create_test_cases_for_multiplication_of_real_and_vector), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vector(double d, Vector2D v, Vector2D expected) { Vector2D actual = d * v; @@ -148,7 +148,7 @@ public void Vector2D_op_Multiplication_double_Vector2D___valid_args___scaled_vec } [TestMethod] - [DynamicData(nameof(Create_test_cases_for_scalar_product_of_two_vectors), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DynamicData(nameof(Create_test_cases_for_scalar_product_of_two_vectors), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void Vector2D_op_Multiplication_Vector2D_Vector2D___valid_Vector2D___double_scalar_product(Vector2D v, Vector2D w, double expected) { double actual = v * w; @@ -226,7 +226,7 @@ public override string ToString() #region https://github.com/microsoft/testfx/issues/1588 [TestMethod] - [DynamicData(nameof(GetTestData), IdentifierStrategy = TestDataIdentifierStrategy.DataIndex)] + [DynamicData(nameof(GetTestData), UnfoldingStrategy = TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex)] public void TestReadonlyCollectionData(string someString, MyData foo) { } From 987325ac85b903dc675154ce1b76dac46e281914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 29 Dec 2024 16:24:58 +0100 Subject: [PATCH 6/6] Apply suggestions from code review --- .../TestFramework/PublicAPI/PublicAPI.Unshipped.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt index 91031aad64..bd2ef498a5 100644 --- a/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/TestFramework/TestFramework/PublicAPI/PublicAPI.Unshipped.txt @@ -1,24 +1,11 @@ #nullable enable -Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.DataRowAttribute.IdentifierStrategy.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType) -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType) -> void -Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.IdentifierStrategy.set -> void Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.AutoDetect = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void *REMOVED*Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataAttribute.DynamicDataAttribute(string! dynamicDataSourceName, System.Type! dynamicDataDeclaringType, Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType dynamicDataSourceType = Microsoft.VisualStudio.TestTools.UnitTesting.DynamicDataSourceType.Property) -> void -Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataIdentifierStrategy.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy.Auto = 0 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy.DataContractSerialization = 1 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy.DataIndex = 2 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.IdentifierStrategy.get -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy -Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy identifierStrategy) -> void -Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceOptionsAttribute.TestDataSourceOptionsAttribute(Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy unfoldingStrategy, Microsoft.VisualStudio.TestTools.UnitTesting.TestDataIdentifierStrategy identifierStrategy) -> void Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataContractJsonSerializer = 3 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy.UnfoldUsingDataIndex = 4 -> Microsoft.VisualStudio.TestTools.UnitTesting.TestDataSourceUnfoldingStrategy static Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Throws(System.Action! action, string! message = "", params object![]! messageArgs) -> TException!