From d3715b03afc7b1af3908c5a551c9254c7751196b Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 21 Feb 2019 11:40:08 -0800 Subject: [PATCH 1/4] Make xUnit test run sequentially to rule out race conditions around 'powershell.config.json' --- build.psm1 | 20 +++++++------------- test/xUnit/xUnit.tests.csproj | 8 ++++++++ test/xUnit/xunit.runner.json | 3 +++ 3 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 test/xUnit/xunit.runner.json diff --git a/build.psm1 b/build.psm1 index 1d0122bd225..a019397e6da 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1374,7 +1374,7 @@ function Test-PSPesterResults function Start-PSxUnit { [CmdletBinding()]param( - [string] $ParallelTestResultsFile = "ParallelXUnitResults.xml" + [string] $xUnitTestResultsFile = "xUnitResults.xml" ) # Add .NET CLI tools to PATH @@ -1422,21 +1422,15 @@ function Start-PSxUnit { dotnet build --configuration $Options.configuration - if (Test-Path $ParallelTestResultsFile) { - Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue + if (Test-Path $xUnitTestResultsFile) { + Remove-Item $xUnitTestResultsFile -Force -ErrorAction SilentlyContinue } - # we are having intermittent issues on macOS with these tests failing. - # VSTS has suggested forcing them to be sequential - if($env:TF_BUILD -and $IsMacOS) - { - Write-Log 'Forcing parallel xunit tests to run sequentially.' - dotnet test -p:ParallelizeTestCollections=false --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" - } else { - dotnet test --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile" - } + # We run the xUnit tests sequentially to avoid race conditions caused by manipulating the config.json file. + # xUnit tests run in parallel by default. To make them run sequentially, we need to define the 'xunit.runner.json' file. + dotnet test --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$xUnitTestResultsFile" - Publish-TestResults -Path $ParallelTestResultsFile -Type 'XUnit' -Title 'Xunit Parallel' + Publish-TestResults -Path $xUnitTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential' } finally { Pop-Location diff --git a/test/xUnit/xUnit.tests.csproj b/test/xUnit/xUnit.tests.csproj index 11704265ae9..227405930ea 100644 --- a/test/xUnit/xUnit.tests.csproj +++ b/test/xUnit/xUnit.tests.csproj @@ -27,4 +27,12 @@ + + + + xunit.runner.json + PreserveNewest + PreserveNewest + + diff --git a/test/xUnit/xunit.runner.json b/test/xUnit/xunit.runner.json new file mode 100644 index 00000000000..f5cdb055132 --- /dev/null +++ b/test/xUnit/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "parallelizeTestCollections": false +} From abde6d2d4904f39d8c44b17232d7dacd78662c50 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 21 Feb 2019 12:10:03 -0800 Subject: [PATCH 2/4] Address Aditya's comment --- build.psm1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build.psm1 b/build.psm1 index a019397e6da..5a8d81780c5 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1420,15 +1420,13 @@ function Start-PSxUnit { } } - dotnet build --configuration $Options.configuration - if (Test-Path $xUnitTestResultsFile) { Remove-Item $xUnitTestResultsFile -Force -ErrorAction SilentlyContinue } # We run the xUnit tests sequentially to avoid race conditions caused by manipulating the config.json file. # xUnit tests run in parallel by default. To make them run sequentially, we need to define the 'xunit.runner.json' file. - dotnet test --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$xUnitTestResultsFile" + dotnet test --configuration $Options.configuration --test-adapter-path:. "--logger:xunit;LogFilePath=$xUnitTestResultsFile" Publish-TestResults -Path $xUnitTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential' } From 82f44a38b52c91aa193ac9e46f7f4febc03e81f9 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 21 Feb 2019 13:49:31 -0800 Subject: [PATCH 3/4] Fix the parameter name in ci.psm1 --- tools/ci.psm1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 26c0718730a..1034124b954 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -288,17 +288,17 @@ function Invoke-CIxUnit throw "CoreCLR pwsh.exe was not built" } - $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml" + $xUnitTestResultsFile = "$pwd\xUnitTestResults.xml" - Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile + Start-PSxUnit -xUnitTestResultsFile $xUnitTestResultsFile Write-Host -ForegroundColor Green 'Uploading PSxUnit test results' - Update-TestResults -resultsFile $ParallelXUnitTestResultsFile - Push-Artifact -Path $ParallelXUnitTestResultsFile -name xunit + Update-TestResults -resultsFile $xUnitTestResultsFile + Push-Artifact -Path $xUnitTestResultsFile -name xunit if(!$SkipFailing.IsPresent) { # Fail the build, if tests failed - Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile + Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile } } @@ -768,10 +768,10 @@ function Invoke-LinuxTests } try { - $ParallelXUnitTestResultsFile = "$pwd/ParallelXUnitTestResults.xml" - Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile + $xUnitTestResultsFile = "$pwd/xUnitTestResults.xml" + Start-PSxUnit -xUnitTestResultsFile $xUnitTestResultsFile # If there are failures, Test-XUnitTestResults throws - Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile + Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile } catch { $result = "FAIL" if (!$resultError) From 7a2833f801d7b26955bffd0e0a5f766e1318b328 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 21 Feb 2019 14:47:48 -0800 Subject: [PATCH 4/4] Update windows.yml to use the new unit test result file name --- .vsts-ci/windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index dab9f6b295f..d2d94b98234 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -90,8 +90,8 @@ jobs: - powershell: | Import-Module .\tools\ci.psm1 - $ParallelXUnitTestResultsFile = "$(System.ArtifactsDirectory)\xunit\ParallelXUnitTestResults.xml" + $xUnitTestResultsFile = "$(System.ArtifactsDirectory)\xunit\xUnitTestResults.xml" - Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile + Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile displayName: Test condition: succeeded()