From d0ddf5def3ea7b6a0cec2cd739354bf4b64c46e5 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 22 Aug 2017 11:53:43 -0700 Subject: [PATCH 1/4] Add ability to verify a test run using the -passthru object from pester --- build.psm1 | 117 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 25 deletions(-) diff --git a/build.psm1 b/build.psm1 index 84f3913f79e..3ae06140aee 100644 --- a/build.psm1 +++ b/build.psm1 @@ -802,7 +802,7 @@ function Publish-PSTestTools { } function Start-PSPester { - [CmdletBinding()] + [CmdletBinding(DefaultParameterSetName='default')] param( [string]$OutputFormat = "NUnitXml", [string]$OutputFile = "pester-tests.xml", @@ -813,8 +813,10 @@ function Start-PSPester { [string]$binDir = (Split-Path (New-PSOptions).Output), [string]$powershell = (Join-Path $binDir 'powershell'), [string]$Pester = ([IO.Path]::Combine($binDir, "Modules", "Pester")), + [Parameter(ParameterSetName='Unelevate',Mandatory=$true)] [switch]$Unelevate, [switch]$Quiet, + [Parameter(ParameterSetName='PassThru',Mandatory=$true)] [switch]$PassThru ) @@ -845,7 +847,7 @@ function Start-PSPester { } Write-Verbose "Running pester tests at '$path' with tag '$($Tag -join ''', ''')' and ExcludeTag '$($ExcludeTag -join ''', ''')'" -Verbose - Publish-PSTestTools + Publish-PSTestTools | ForEach-Object {Write-Host $_} # All concatenated commands/arguments are suffixed with the delimiter (space) $Command = "" @@ -920,7 +922,23 @@ function Start-PSPester { } else { - & $powershell -noprofile -c $Command + if($PassThru.IsPresent) + { + $passThruFile = [System.IO.Path]::GetTempFileName() + try { + $Command += "|Export-Clixml -Path '$passThruFile' -Force" + Start-NativeExecution -sb {& $powershell -noprofile -c $Command} | ForEach-Object { Write-Host $_} + Import-Clixml -Path $passThruFile | Where-Object {$_.TotalCount} + } + finally + { + Remove-Item $passThruFile -ErrorAction SilentlyContinue + } + } + else + { + Start-NativeExecution -sb {& $powershell -noprofile -c $Command} + } } } finally { $env:PSModulePath = $originalModulePath @@ -952,13 +970,40 @@ function script:Start-UnelevatedProcess function Show-PSPesterError { - param ( [Xml.XmlElement]$testFailure ) - logerror ("Description: " + $testFailure.description) - logerror ("Name: " + $testFailure.name) + [CmdletBinding(DefaultParameterSetName='xml')] + param ( + [Parameter(ParameterSetName='xml',Mandatory)] + [Xml.XmlElement]$testFailure, + [Parameter(ParameterSetName='object',Mandatory)] + [PSCustomObject]$testFailureObject + ) + + if($PSCmdLet.ParameterSetName -eq 'file') + { + $description = $testFailure.description + $name = $testFailure.name + $message = $testFailure.failure.message + $stackTrace = $testFailure.failure."stack-trace" + } + elseif($PSCmdLet.ParameterSetName -eq 'object') + { + $description = $testFailureObject.Describe + '/' + $testFailureObject.Context + $name = $testFailureObject.Name + $message = $testFailureObject.FailureMessage + $stackTrace = $testFailureObject.StackTrace + } + else + { + throw 'Unknow Show-PSPester parameter set' + } + + logerror ("Description: " + $description) + logerror ("Name: " + $name) logerror "message:" - logerror $testFailure.failure.message + logerror $message logerror "stack-trace:" - logerror $testFailure.failure."stack-trace" + logerror $stackTrace + } # @@ -966,32 +1011,54 @@ function Show-PSPesterError # Throw if a test failed function Test-PSPesterResults { + [CmdletBinding(DefaultParameterSetName='file')] param( + [Parameter(ParameterSetName='file')] [string]$TestResultsFile = "pester-tests.xml", - [string]$TestArea = 'test/powershell' - ) + [Parameter(ParameterSetName='file')] + [string]$TestArea = 'test/powershell', + [Parameter(ParameterSetName='PesterPassThruObject',Mandatory)] + [pscustomobject] $ResultObject + ) - if(!(Test-Path $TestResultsFile)) + if($PSCmdLet.ParameterSetName -eq 'file') { - throw "Test result file '$testResultsFile' not found for $TestArea." - } + if(!(Test-Path $TestResultsFile)) + { + throw "Test result file '$testResultsFile' not found for $TestArea." + } - $x = [xml](Get-Content -raw $testResultsFile) - if ([int]$x.'test-results'.failures -gt 0) - { - logerror "TEST FAILURES" - # switch between methods, SelectNode is not available on dotnet core - if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] ) { - $failures = [System.Xml.XmlDocumentXPathExtensions]::SelectNodes($x."test-results",'.//test-case[@result = "Failure"]') + $x = [xml](Get-Content -raw $testResultsFile) + if ([int]$x.'test-results'.failures -gt 0) + { + logerror "TEST FAILURES" + # switch between methods, SelectNode is not available on dotnet core + if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] ) { + $failures = [System.Xml.XmlDocumentXPathExtensions]::SelectNodes($x."test-results",'.//test-case[@result = "Failure"]') + } + else { + $failures = $x.SelectNodes('.//test-case[@result = "Failure"]') + } + foreach ( $testfail in $failures ) + { + Show-PSPesterError $testfail + } + throw "$($x.'test-results'.failures) tests in $TestArea failed" } - else { - $failures = $x.SelectNodes('.//test-case[@result = "Failure"]') + } + elseif($PSCmdLet.ParameterSetName -eq 'PesterPassThruObject') + { + if($ResultObject.TotalCount -le 0) + { + logerror 'NO TESTS RUN' } - foreach ( $testfail in $failures ) + elseif($ResultObject.FailedCount -gt 0) { - Show-PSPesterError $testfail + logerror 'TEST FAILURES' + $ResultObject.TestResult | Where-Object {$_.Passed -eq $false} | ForEach-Object { + Show-PSPesterError -testFailureObject $_ + } } - throw "$($x.'test-results'.failures) tests in $TestArea failed" } } From 37732831ed53f3f7d331c1bffcea62005012acc7 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 22 Aug 2017 11:54:14 -0700 Subject: [PATCH 2/4] update travis-ci to use -passthru object to verify test results --- tools/travis.ps1 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/travis.ps1 b/tools/travis.ps1 index 4b681dc69ed..7fb945ab0f4 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -149,7 +149,10 @@ else $ProgressPreference = $originalProgressPreference } - $pesterParam = @{ 'binDir' = $output } + $pesterParam = @{ + 'binDir' = $output + 'PassThru' = $true + } if ($isFullBuild) { $pesterParam['Tag'] = @('CI','Feature','Scenario') @@ -165,7 +168,7 @@ else Remove-Item -force ${telemetrySemaphoreFilepath} } - Start-PSPester @pesterParam + $pesterPassThruObject = Start-PSPester @pesterParam if (-not $isPr) { # Run 'CrossGen' for push commit, so that we can generate package. @@ -197,7 +200,7 @@ else try { # this throws if there was an error - Test-PSPesterResults + Test-PSPesterResults -ResultObject $pesterPassThruObject $result = "PASS" } catch { From cc0452514ad24a3be7e8da1edb44e9907640b4ab Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 22 Aug 2017 16:02:45 -0700 Subject: [PATCH 3/4] Address PR feedback --- build.psm1 | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/build.psm1 b/build.psm1 index 3ae06140aee..c444244d048 100644 --- a/build.psm1 +++ b/build.psm1 @@ -922,13 +922,14 @@ function Start-PSPester { } else { - if($PassThru.IsPresent) + if ($PassThru.IsPresent) { $passThruFile = [System.IO.Path]::GetTempFileName() - try { + try + { $Command += "|Export-Clixml -Path '$passThruFile' -Force" Start-NativeExecution -sb {& $powershell -noprofile -c $Command} | ForEach-Object { Write-Host $_} - Import-Clixml -Path $passThruFile | Where-Object {$_.TotalCount} + Import-Clixml -Path $passThruFile | Where-Object {$_.TotalCount -is [Int32]} } finally { @@ -978,14 +979,14 @@ function Show-PSPesterError [PSCustomObject]$testFailureObject ) - if($PSCmdLet.ParameterSetName -eq 'file') + if ($PSCmdLet.ParameterSetName -eq 'xml') { $description = $testFailure.description $name = $testFailure.name $message = $testFailure.failure.message $stackTrace = $testFailure.failure."stack-trace" } - elseif($PSCmdLet.ParameterSetName -eq 'object') + elseif ($PSCmdLet.ParameterSetName -eq 'object') { $description = $testFailureObject.Describe + '/' + $testFailureObject.Context $name = $testFailureObject.Name @@ -994,7 +995,7 @@ function Show-PSPesterError } else { - throw 'Unknow Show-PSPester parameter set' + throw 'Unknown Show-PSPester parameter set' } logerror ("Description: " + $description) @@ -1033,10 +1034,12 @@ function Test-PSPesterResults { logerror "TEST FAILURES" # switch between methods, SelectNode is not available on dotnet core - if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] ) { + if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] ) + { $failures = [System.Xml.XmlDocumentXPathExtensions]::SelectNodes($x."test-results",'.//test-case[@result = "Failure"]') } - else { + else + { $failures = $x.SelectNodes('.//test-case[@result = "Failure"]') } foreach ( $testfail in $failures ) @@ -1046,13 +1049,13 @@ function Test-PSPesterResults throw "$($x.'test-results'.failures) tests in $TestArea failed" } } - elseif($PSCmdLet.ParameterSetName -eq 'PesterPassThruObject') + elseif ($PSCmdLet.ParameterSetName -eq 'PesterPassThruObject') { - if($ResultObject.TotalCount -le 0) + if ($ResultObject.TotalCount -le 0) { logerror 'NO TESTS RUN' } - elseif($ResultObject.FailedCount -gt 0) + elseif ($ResultObject.FailedCount -gt 0) { logerror 'TEST FAILURES' $ResultObject.TestResult | Where-Object {$_.Passed -eq $false} | ForEach-Object { From 996c4af77aecea15bd6be0d95e9d60073d4a6406 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 22 Aug 2017 16:55:01 -0700 Subject: [PATCH 4/4] call show-pspestererror with a named parameter --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index c444244d048..5ffb21e1107 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1044,7 +1044,7 @@ function Test-PSPesterResults } foreach ( $testfail in $failures ) { - Show-PSPesterError $testfail + Show-PSPesterError -testFailure $testfail } throw "$($x.'test-results'.failures) tests in $TestArea failed" }