From a7f5131a552e612666e656f794b357c9e4dc3707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 30 Aug 2024 10:49:41 +0200 Subject: [PATCH 1/2] Collect test assets on failed tests --- azure-pipelines.yml | 14 +++++++++ .../NativeAotTests.cs | 30 ++++++------------- .../SdkTests.cs | 28 +++++------------ 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2e9eaf639f..d5fee05266 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -116,6 +116,13 @@ stages: displayName: Upload coverage to codecov.io condition: and(succeeded(), eq(variables._BuildConfig, 'Debug')) + - task: PublishBuildArtifacts@1 + displayName: 'Publish test assets temp folders' + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite' + ArtifactName: TmpArtifacts_Windows_$(_BuildConfig) + condition: failed() + # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx - task: PublishBuildArtifacts@1 @@ -153,3 +160,10 @@ stages: ./test.sh --configuration $(_BuildConfig) --ci --test --integrationTest --nobl name: Test displayName: Tests + + - task: PublishBuildArtifacts@1 + displayName: 'Publish test assets temp folders' + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite' + ArtifactName: TmpArtifacts_Linux_$(_BuildConfig) + condition: failed() diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs index db2396cba1..26569fc5cb 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs @@ -91,11 +91,6 @@ public NativeAotTests(ITestExecutionContext testExecutionContext, AcceptanceFixt public async Task NativeAotTests_WillRunWithExitCodeZero() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return; - } - using TestAsset generator = await TestAsset.GenerateAssetAsync( "NativeAotTests", SourceCode @@ -106,23 +101,16 @@ public async Task NativeAotTests_WillRunWithExitCodeZero() .PatchCodeWithReplace("$MSTestEngineVersion$", MSTestEngineVersion), addPublicFeeds: true); - // The native AOT publication is pretty flaky and is often failing on CI with "fatal error LNK1136: invalid or corrupt file", - // or sometimes doesn't fail but the native code generation is not done. - // So, we retry the publication a few times. - await RetryHelper.RetryAsync( - async () => - { - await DotnetCli.RunAsync( - $"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", - _acceptanceFixture.NuGetGlobalPackagesFolder.Path, - retryCount: 0); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( - $"publish -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", - _acceptanceFixture.NuGetGlobalPackagesFolder.Path, - retryCount: 0); - compilationResult.AssertOutputContains("Generating native code"); - }, times: 15, every: TimeSpan.FromSeconds(5)); + await DotnetCli.RunAsync( + $"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", + _acceptanceFixture.NuGetGlobalPackagesFolder.Path, + retryCount: 0); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( + $"publish -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", + _acceptanceFixture.NuGetGlobalPackagesFolder.Path, + retryCount: 0); + compilationResult.AssertOutputContains("Generating native code"); var testHost = TestHost.LocateFrom(generator.TargetAssetPath, "NativeAotTests", TargetFrameworks.NetCurrent.Arguments, RID, Verb.publish); TestHostResult result = await testHost.ExecuteAsync(); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 51aa940125..ae06bb2b58 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -295,13 +295,8 @@ public async Task Invalid_TestingProfile_Name_Should_Fail(string multiTfm, Build compilationResult.AssertOutputContains("Invalid value for property TestingExtensionsProfile. Valid values are 'Default', 'AllMicrosoft' and 'None'."); } - public async Task NativeAot_Smoke_Test_Windows() + public async Task NativeAot_Smoke_Test() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return; - } - using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode @@ -313,20 +308,13 @@ public async Task NativeAot_Smoke_Test_Windows() """), addPublicFeeds: true); - // The native AOT publication is pretty flaky and is often failing on CI with "fatal error LNK1136: invalid or corrupt file", - // or sometimes doesn't fail but the native code generation is not done. - // So, we retry the publication a few times. - await RetryHelper.RetryAsync( - async () => - { - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( - $"publish -r {RID} -f {TargetFrameworks.NetCurrent.Arguments} {testAsset.TargetAssetPath}", - _acceptanceFixture.NuGetGlobalPackagesFolder.Path, - // We prefer to use the outer retry mechanism as we need some extra checks - retryCount: 0); - compilationResult.AssertOutputContains("Generating native code"); - compilationResult.AssertOutputNotContains("warning"); - }, times: 15, every: TimeSpan.FromSeconds(5)); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( + $"publish -r {RID} -f {TargetFrameworks.NetCurrent.Arguments} {testAsset.TargetAssetPath}", + _acceptanceFixture.NuGetGlobalPackagesFolder.Path, + // We prefer to use the outer retry mechanism as we need some extra checks + retryCount: 0); + compilationResult.AssertOutputContains("Generating native code"); + compilationResult.AssertOutputNotContains("warning"); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent.Arguments, verb: Verb.publish); TestHostResult testHostResult = await testHost.ExecuteAsync(); From fcc1ff50f1c525749198687383a9f989cca48a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 30 Aug 2024 13:15:16 +0200 Subject: [PATCH 2/2] Fix IDEXXXX issues --- .../MSTest.Acceptance.IntegrationTests/NativeAotTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs index 26569fc5cb..cd19324c41 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Runtime.InteropServices; - using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; @@ -101,7 +99,6 @@ public async Task NativeAotTests_WillRunWithExitCodeZero() .PatchCodeWithReplace("$MSTestEngineVersion$", MSTestEngineVersion), addPublicFeeds: true); - await DotnetCli.RunAsync( $"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}", _acceptanceFixture.NuGetGlobalPackagesFolder.Path,