From 592a59f725cc75291f3e7b9fe2fa75444c3cced8 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 11 Dec 2017 18:27:54 -0800 Subject: [PATCH 01/10] Add ability for get-psoptions to default to new-psoptions --- build.psm1 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/build.psm1 b/build.psm1 index ae3e4dfc6fd..54675dcd46f 100644 --- a/build.psm1 +++ b/build.psm1 @@ -824,6 +824,17 @@ function New-PSOptions { # Get the Options of the last build function Get-PSOptions { + param( + [Parameter(HelpMessage='Defaults to New-PSOption if a build has not ocurred.')] + [switch] + $DefaultToNew + ) + + if(!$script:Options -and $DefaultToNew.IsPresent) + { + return New-PSOptions + } + return $script:Options } From b9db2bec55e01ff2426a80e34e6bf446e18dd272 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 11 Dec 2017 18:29:14 -0800 Subject: [PATCH 02/10] refactor code to restore pester into separate function --- build.psm1 | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index 54675dcd46f..4f602148937 100644 --- a/build.psm1 +++ b/build.psm1 @@ -660,9 +660,21 @@ function Restore-PSModuleToBuild if($CI.IsPresent) { - Restore-GitModule -Destination $modulesDir -Uri 'https://github.com/PowerShell/psl-pester' -Name Pester -CommitSha 'aa243108e7da50a8cf82513b6dd649b653c70b0e' + Restore-PSPester -Destination $modulesDir } } + +function Restore-PSPester +{ + param + ( + [ValidateNotNullOrEmpty()] + [string] + $Destination = ([IO.Path]::Combine((Split-Path (Get-PSOptions -DefaultToNew).Output), "Modules")) + ) + + Restore-GitModule -Destination $Destination -Uri 'https://github.com/PowerShell/psl-pester' -Name Pester -CommitSha 'aa243108e7da50a8cf82513b6dd649b653c70b0e' +} function Compress-TestContent { [CmdletBinding()] param( From d68aef558d8739f64e55050bd09807e45533805f Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 11 Dec 2017 18:31:00 -0800 Subject: [PATCH 03/10] fix issue with publish-pstesttools when a build has not been run since build.psm1 has been imported (try to use the default options) --- build.psm1 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build.psm1 b/build.psm1 index 4f602148937..f4674a9bce7 100644 --- a/build.psm1 +++ b/build.psm1 @@ -939,10 +939,8 @@ function Publish-PSTestTools { @{Path="${PSScriptRoot}/test/tools/TestExe";Output="testexe"} @{Path="${PSScriptRoot}/test/tools/WebListener";Output="WebListener"} ) - if ($null -eq $Options) - { - $Options = New-PSOptions - } + + $Options = Get-PSOptions -DefaultToNew # Publish tools so it can be run by tests foreach ($tool in $tools) From 78e174928fe73fc62a60ca0c19bfc424986623f0 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 11 Dec 2017 18:31:57 -0800 Subject: [PATCH 04/10] make start-pspester use the last build, not just use the default options --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index f4674a9bce7..294a9db1bc4 100644 --- a/build.psm1 +++ b/build.psm1 @@ -968,7 +968,7 @@ function Start-PSPester { [string[]]$Tag = @("CI","Feature"), [string[]]$Path = @("$PSScriptRoot/test/common","$PSScriptRoot/test/powershell"), [switch]$ThrowOnFailure, - [string]$binDir = (Split-Path (New-PSOptions).Output), + [string]$binDir = (Split-Path (Get-PSOptions -DefaultToNew).Output), [string]$powershell = (Join-Path $binDir 'pwsh'), [string]$Pester = ([IO.Path]::Combine($binDir, "Modules", "Pester")), [Parameter(ParameterSetName='Unelevate',Mandatory=$true)] From 2bacfd92165dac327c97f5a5b20b99dd59b3e912 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 11 Dec 2017 18:33:38 -0800 Subject: [PATCH 05/10] fix issue in restore caused some files not to be removed --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index 294a9db1bc4..a2cc22e60eb 100644 --- a/build.psm1 +++ b/build.psm1 @@ -2435,7 +2435,7 @@ function Restore-GitModule $gitItems = Join-Path -Path $clonePath -ChildPath '.git*' $ymlItems = Join-Path -Path $clonePath -ChildPath '*.yml' - Get-ChildItem -Path $gitItems, $ymlItems | Remove-Item -Recurse -Force + Get-ChildItem -attributes hidden,normal -Path $gitItems, $ymlItems | Remove-Item -Recurse -Force } finally { From dd78ca5b65eb744933875c09a7a30eabeed18e09 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 11 Dec 2017 18:34:14 -0800 Subject: [PATCH 06/10] update message no what to do when pester is missing --- build.psm1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.psm1 b/build.psm1 index a2cc22e60eb..f89c7fe4013 100644 --- a/build.psm1 +++ b/build.psm1 @@ -984,8 +984,8 @@ function Start-PSPester { { Write-Warning @" Pester module not found. -Make sure that the proper git submodules are installed by running: - git submodule update --init +Restore to module to '$Pester' by running: + Restore-PSPester "@ return; } From 240a995afdbef82f986785a54f6c6b6235bfca21 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 12 Dec 2017 09:52:10 -0800 Subject: [PATCH 07/10] make markdown files pass tests --- docs/git/submodules.md | 6 ++---- docs/testing-guidelines/testing-guidelines.md | 15 +++++++-------- test/common/markdown/markdown.tests.ps1 | 3 +++ test/powershell/README.md | 15 +++++---------- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/docs/git/submodules.md b/docs/git/submodules.md index 7464a1ef86c..79f880ee91f 100644 --- a/docs/git/submodules.md +++ b/docs/git/submodules.md @@ -1,5 +1,4 @@ -Submodules -========== +# Submodules While most developers will not have to deal with submodules on a regular basis, those who do should read this information. The submodules currently in this project are: @@ -11,8 +10,7 @@ The submodules currently in this project are: [submodules]: https://www.git-scm.com/book/en/v2/Git-Tools-Submodules -Rebase and Fast-Forward Merge Pull Requests in Submodules -========================================================= +## Rebase and Fast-Forward Merge Pull Requests in Submodules *This is not necessary in the superproject, only submodules!* diff --git a/docs/testing-guidelines/testing-guidelines.md b/docs/testing-guidelines/testing-guidelines.md index a4de43c64c0..a036cab2e6a 100755 --- a/docs/testing-guidelines/testing-guidelines.md +++ b/docs/testing-guidelines/testing-guidelines.md @@ -1,10 +1,10 @@ - # Testing Guidelines Testing is a critical and required part of the PowerShell project. The Microsoft PowerShell team created nearly 100,000 tests over the last 12 years which we run as part of the release process for Windows PowerShell. -Having all of those tests available for the initial release of PowerShell was not feasible, and we have targeted those tests which we believe will provide us the ability to catch regressions in the areas which have had the largest changes for PowerShell. +Having all of those tests available for the initial release of PowerShell was not feasible, +and we have targeted those tests which we believe will provide us the ability to catch regressions in the areas which have had the largest changes for PowerShell. It is our intent to continue to release more and more of our tests until we have the coverage we need. For creating new tests, please review the [documents](https://github.com/PowerShell/PowerShell/tree/master/docs/testing-guidelines) on how to create tests for PowerShell. @@ -69,7 +69,8 @@ The Pester framework allows `Describe` blocks to be tagged, and our CI system re One of the following tags must be used: * `CI` - this tag indicates that the tests in the `Describe` block will be executed as part of the CI/PR process -* `Feature` - tests with this tag will not be executed as part of the CI/PR process, but they will be executed on a daily basis as part of a `cron` driven build. They indicate that the test will be validating more behavior, or will be using remote network resources (ex: package management tests) +* `Feature` - tests with this tag will not be executed as part of the CI/PR process, but they will be executed on a daily basis as part of a `cron` driven build. + They indicate that the test will be validating more behavior, or will be using remote network resources (ex: package management tests) * `Scenario` - this tag indicates a larger scale test interacting with multiple areas of functionality and/or remote resources, these tests are also run daily. Additionally, the tag: @@ -124,20 +125,20 @@ environment is not the default or has any customization. For example, to run all the Pester tests for CI (assuming you are at the root of the PowerShell repo): -``` +```PowerShell Import-Module ./build.psm1 Start-PSPester ``` If you wish to run specific tests, that is possible as well: -``` +```PowerShell Start-PSPester -Path test/powershell/engine/Api ``` Or a specific Pester test file: -``` +```PowerShell Start-PSPester -Path test/powershell/engine/Api/XmlAdapter.Tests.ps1 ``` @@ -148,8 +149,6 @@ in Microsoft's internal test frameworks. The tests that you created for your change and the library of historical tests will be run to determine if any regressions are present. If these tests find regressions, you'll be notified that your PR is not ready, and provided with enough information to investigate why the failure happened. - - ## Test Layout We have taken a functional approach to the layout of our Pester tests and you should place new tests in their appropriate location. diff --git a/test/common/markdown/markdown.tests.ps1 b/test/common/markdown/markdown.tests.ps1 index 43a88444786..00bfd7cdcac 100644 --- a/test/common/markdown/markdown.tests.ps1 +++ b/test/common/markdown/markdown.tests.ps1 @@ -76,10 +76,13 @@ Describe 'Common Tests - Validate Markdown Files' -Tag 'CI' { './docs/*.md' './docs/building/*.md' './docs/cmdlet-example/*.md' + './docs/git/submodules.md' './docs/installation/*.md' './docs/maintainers/README.md' + './docs/testing-guidelines/testing-guidelines.md' './demos/SSHRemoting/*.md' './docker/*.md' + './test/powershell/README.md' './tools/*.md' './.github/CONTRIBUTING.md' ) diff --git a/test/powershell/README.md b/test/powershell/README.md index 193cb5ac022..4e8f1f4c78e 100644 --- a/test/powershell/README.md +++ b/test/powershell/README.md @@ -1,19 +1,16 @@ -Pester Testing Test Guide -========================= +# Pester Testing Test Guide Also see the [Writing Pester Tests](../../docs/testing-guidelines/WritingPesterTests.md) document. -Running Pester Tests --------------------- +## Running Pester Tests Go to the top level of the PowerShell repository and run: `Start-PSPester` inside a self-hosted copy of PowerShell. You can use `Start-PSPester -Tests SomeTestSuite*` to limit the tests run. -Testing new `powershell` processes ----------------------------------- +## Testing new `powershell` processes Any launch of a new `powershell` process must include `-noprofile` so that modified user and system profiles do not causes tests to fail. You also must @@ -27,8 +24,7 @@ Example: & $powershell -noprofile -command "ExampleCommand" | Should Be "ExampleOutput" ``` -Portability ------------ +## Portability Some tests simply must be tied to certain platforms. Use Pester's `-Skip` directive on an `It` statement to do this. For instance to run @@ -44,8 +40,7 @@ Or only on Linux and OS X: It "Should do something on Linux" -Skip:$IsWindows { ... } ``` -Pending -------- +## Pending When writing a test that should pass, but does not, please do not skip or delete the test, but use `It "Should Pass" -Pending` to mark the test as pending, and From c2d0fca75ea9620aded2dccc5452464abbbb015f Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 12 Dec 2017 09:52:46 -0800 Subject: [PATCH 08/10] update docs to mention to get pester using restore-PSPester instead of submodule --- docs/git/submodules.md | 2 -- docs/testing-guidelines/testing-guidelines.md | 10 +++------- test/powershell/README.md | 4 +++- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/git/submodules.md b/docs/git/submodules.md index 79f880ee91f..621fe275f96 100644 --- a/docs/git/submodules.md +++ b/docs/git/submodules.md @@ -3,8 +3,6 @@ While most developers will not have to deal with submodules on a regular basis, those who do should read this information. The submodules currently in this project are: -- `src/Modules/Pester`: The Pester testing framework for PowerShell - - `src/libpsl-native/test/googletest`: The GoogleTest framework for Linux native code diff --git a/docs/testing-guidelines/testing-guidelines.md b/docs/testing-guidelines/testing-guidelines.md index a036cab2e6a..92072071e94 100755 --- a/docs/testing-guidelines/testing-guidelines.md +++ b/docs/testing-guidelines/testing-guidelines.md @@ -106,19 +106,15 @@ Currently, we have a minuscule number of tests which are run by using xUnit. ## Running tests outside of CI When working on new features or fixes, it is natural to want to run those tests locally before making a PR. -Two helper functions are part of the build.psm1 module to help with that: +Three helper functions are part of the build.psm1 module to help with that: +* `Restore-PSPester` will restore Pester, which is needed to run `Start-PSPester` * `Start-PSPester` will execute all Pester tests which are run by the CI system * `Start-PSxUnit` will execute the available xUnit tests run by the CI system Our CI system runs these as well; there should be no difference between running these on your dev system, versus in CI. -Make sure that the git submodules have been loaded into your project before running `Start-PSPester`, or it will fail to run. -If you did not clone the project with the `--recursive` flag, you can load the submodules by running: - -``` -git submodule update --init -``` +Make sure that you run `Restore-PSPester` before running `Start-PSPester`, or it will fail to run. When running tests in this way, be sure that you have started PowerShell with `-noprofile` as some tests will fail if the environment is not the default or has any customization. diff --git a/test/powershell/README.md b/test/powershell/README.md index 4e8f1f4c78e..5fd340a5e5c 100644 --- a/test/powershell/README.md +++ b/test/powershell/README.md @@ -5,7 +5,9 @@ document. ## Running Pester Tests -Go to the top level of the PowerShell repository and run: `Start-PSPester` +First, restore the correct version of Pester using `Restore-PSPester`. + +Then, go to the top level of the PowerShell repository and run: `Start-PSPester` inside a self-hosted copy of PowerShell. You can use `Start-PSPester -Tests SomeTestSuite*` to limit the tests run. From 20df4d823989a464b99389adea055d690241c773 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 12 Dec 2017 11:01:38 -0800 Subject: [PATCH 09/10] fix grammar in error message --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index f89c7fe4013..0fed7d091dd 100644 --- a/build.psm1 +++ b/build.psm1 @@ -984,7 +984,7 @@ function Start-PSPester { { Write-Warning @" Pester module not found. -Restore to module to '$Pester' by running: +Restore the module to '$Pester' by running: Restore-PSPester "@ return; From 17a1f6c76aa4aaa303f0e9299522d1ea05c4f2b7 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 12 Dec 2017 14:04:27 -0800 Subject: [PATCH 10/10] fix test failure --- docs/git/submodules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/git/submodules.md b/docs/git/submodules.md index 621fe275f96..52acc6ad5f5 100644 --- a/docs/git/submodules.md +++ b/docs/git/submodules.md @@ -10,7 +10,7 @@ The submodules currently in this project are: ## Rebase and Fast-Forward Merge Pull Requests in Submodules -*This is not necessary in the superproject, only submodules!* +Note: *This is not necessary in the superproject, only submodules!* **DO NOT** commit updates unless absolutely necessary. When submodules must be updated, a separate Pull Request must be submitted, reviewed, and merged before updating the superproject.