diff --git a/.spelling b/.spelling index ab1d9720288..cacbde03054 100644 --- a/.spelling +++ b/.spelling @@ -861,6 +861,11 @@ SytzeAndr yashrajbharti Leonhardt tylerleonhardt +nuget.config +libmi +rpms +authenticode +env - CHANGELOG.md aavdberg asrosent @@ -935,3 +940,4 @@ wpaProfile - CHANGELOG/preview.md ThomasNieto spongemike2 + diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 6c7a94e0216..031316eef26 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -11,7 +11,6 @@ trigger: include: - '*' exclude: - - /tools/releaseBuild/**/* - /.vsts-ci/misc-analysis.yml - /.github/ISSUE_TEMPLATE/* - /.dependabot/config.yml @@ -25,7 +24,8 @@ pr: include: - '*' exclude: - - /tools/releaseBuild/**/* + - tools/releaseBuild/* + - tools/releaseBuild/azureDevOps/templates/* - /.vsts-ci/misc-analysis.yml - /.github/ISSUE_TEMPLATE/* - /.dependabot/config.yml diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index ca4a66fc7e6..96121e80f77 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -25,10 +25,11 @@ pr: include: - '*' exclude: - - /tools/releaseBuild/**/* - /.vsts-ci/misc-analysis.yml - /.github/ISSUE_TEMPLATE/* - /.dependabot/config.yml + - tools/releaseBuild/* + - tools/releaseBuild/azureDevOps/templates/* variables: DOTNET_CLI_TELEMETRY_OPTOUT: 1 diff --git a/.vsts-ci/templates/windows-packaging.yml b/.vsts-ci/templates/windows-packaging.yml index df422b812d5..455efbb9fec 100644 --- a/.vsts-ci/templates/windows-packaging.yml +++ b/.vsts-ci/templates/windows-packaging.yml @@ -27,7 +27,7 @@ jobs: displayName: Bootstrap condition: succeededOrFailed() - - powershell: | + - pwsh: | Import-Module .\tools\ci.psm1 New-CodeCoverageAndTestPackage Invoke-CIFinish -NuGetKey $(NUGET_KEY) diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index 81e74dcbc77..11bccbbbfa6 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -27,6 +27,8 @@ pr: - /.vsts-ci/misc-analysis.yml - /.github/ISSUE_TEMPLATE/* - /.dependabot/config.yml + - tools/releaseBuild/* + - tools/releaseBuild/azureDevOps/templates/* variables: GIT_CONFIG_PARAMETERS: "'core.autocrlf=false'" diff --git a/CHANGELOG/7.0.md b/CHANGELOG/7.0.md index b0130b2c80e..b51667e8264 100644 --- a/CHANGELOG/7.0.md +++ b/CHANGELOG/7.0.md @@ -1,5 +1,48 @@ # 7.0 Changelog +## [7.0.4] - 2021-01-19 + +### Build and Packaging Improvements + +
+ + +Bump .NET SDK to version 3.1.405 + + + + +
+ +[7.0.4]: https://github.com/PowerShell/PowerShell/compare/v7.0.3...v7.0.4 + ## [7.0.3] - 2020-07-16 ### Tests diff --git a/assets/AppxManifest.xml b/assets/AppxManifest.xml index 29a7fe78e26..8d417c1d1a4 100644 --- a/assets/AppxManifest.xml +++ b/assets/AppxManifest.xml @@ -9,7 +9,7 @@ xmlns:desktop6="http://schemas.microsoft.com/appx/manifest/desktop/windows10/6" xmlns:rescap="http://schemas.microsoft.com/appx/manifest/foundation/windows10/restrictedcapabilities"> - + $DISPLAYNAME$ diff --git a/assets/default.help.txt b/assets/default.help.txt new file mode 100644 index 00000000000..a5b5cefc9cd --- /dev/null +++ b/assets/default.help.txt @@ -0,0 +1,108 @@ + +TOPIC + PowerShell Help System + +SHORT DESCRIPTION + Displays help about PowerShell cmdlets and concepts. + +LONG DESCRIPTION + PowerShell Help describes PowerShell cmdlets, functions, scripts, and + modules, and explains concepts, including the elements of the PowerShell + language. + + PowerShell does not include help files, but you can read the help topics + online, or use the Update-Help cmdlet to download help files to your + computer and then use the Get-Help cmdlet to display the help topics at + the command line. + + You can also use the Update-Help cmdlet to download updated help files + as they are released so that your local help content is never obsolete. + + Without help files, Get-Help displays auto-generated help for cmdlets, + functions, and scripts. + + + ONLINE HELP + You can find help for PowerShell online at + https://go.microsoft.com/fwlink/?LinkID=108518. + + To open online help for any cmdlet or function, type: + + Get-Help -Online + + UPDATE-HELP + To download and install help files on your computer: + + 1. Start PowerShell with the "Run as administrator" option. + 2. Type: + + Update-Help + + After the help files are installed, you can use the Get-Help cmdlet to + display the help topics. You can also use the Update-Help cmdlet to + download updated help files so that your local help files are always + up-to-date. + + For more information about the Update-Help cmdlet, type: + + Get-Help Update-Help -Online + + or go to: https://go.microsoft.com/fwlink/?LinkID=210614 + + + GET-HELP + The Get-Help cmdlet displays help at the command line from content in + help files on your computer. Without help files, Get-Help displays basic + help about cmdlets and functions. You can also use Get-Help to display + online help for cmdlets and functions. + + To get help for a cmdlet, type: + + Get-Help + + To get online help, type: + + Get-Help -Online + + The titles of conceptual topics begin with "About_". To get help for a + concept or language element, type: + + Get-Help About_ + + To search for a word or phrase in all help files, type: + + Get-Help + + For more information about the Get-Help cmdlet, type: + + Get-Help Get-Help -Online + + or go to: https://go.microsoft.com/fwlink/?LinkID=113316 + + + EXAMPLES: + Save-Help : Download help files from the internet and save + them on a file share. + + Update-Help : Downloads and installs help files from the + internet or a file share. + + Get-Help Get-Process : Displays help about the Get-Process cmdlet. + + Get-Help Get-Process -Online + : Opens online help for the Get-Process cmdlet. + + Help Get-Process : Displays help about Get-Process one page at a + time. + Get-Process -? : Displays help about the Get-Process cmdlet. + + Get-Help About_Modules : Displays help about PowerShell modules. + + Get-Help remoting : Searches the help topics for the word "remoting." + + SEE ALSO: + about_Updatable_Help + Get-Help + Save-Help + Update-Help + diff --git a/assets/files.wxs b/assets/files.wxs index 1c85237572f..37b5f8b7532 100644 --- a/assets/files.wxs +++ b/assets/files.wxs @@ -1940,24 +1940,6 @@ - - - - - - - - - - - - - - - - - - @@ -3074,8 +3056,8 @@ - - + + @@ -4061,13 +4043,7 @@ - - - - - - - + diff --git a/build.psm1 b/build.psm1 index d6c471cbeb2..7d5fa3e211f 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1,4 +1,4 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. +# Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # On Unix paths is separated by colon @@ -325,7 +325,8 @@ function Start-PSBuild { try { # Excluded sqlite3 folder is due to this Roslyn issue: https://github.com/dotnet/roslyn/issues/23060 # Excluded src/Modules/nuget.config as this is required for release build. - git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3 --exclude src/Modules/nuget.config + # Excluded nuget.config as this is required for release build. + git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3 --exclude src/Modules/nuget.config --exclude nuget.config } finally { Pop-Location } @@ -598,7 +599,11 @@ function Restore-PSPackage [Parameter()] $Options = (Get-PSOptions -DefaultToNew), - [switch] $Force + [switch] $Force, + + [switch] $InteractiveAuth, + + [switch] $PSModule ) if (-not $ProjectDirs) @@ -622,21 +627,28 @@ function Restore-PSPackage 'Microsoft.NET.Sdk' } - if ($Options.Runtime -notlike 'fxdependent*') { + if ($PSModule.IsPresent) { + $RestoreArguments = @("--verbosity") + } + elseif ($Options.Runtime -notlike 'fxdependent*') { $RestoreArguments = @("--runtime", $Options.Runtime, "/property:SDKToUse=$sdkToUse", "--verbosity") } else { $RestoreArguments = @("/property:SDKToUse=$sdkToUse", "--verbosity") } - if ($PSCmdlet.MyInvocation.BoundParameters["Verbose"].IsPresent) { + if ($VerbosePreference -eq 'Continue') { $RestoreArguments += "detailed" } else { $RestoreArguments += "quiet" } + if ($InteractiveAuth) { + $RestoreArguments += "--interactive" + } + $ProjectDirs | ForEach-Object { $project = $_ - Write-Log "Run dotnet restore $project $RestoreArguments" + Write-Log -message "Run dotnet restore $project $RestoreArguments" $retryCount = 0 $maxTries = 5 while($retryCount -lt $maxTries) @@ -647,7 +659,7 @@ function Restore-PSPackage } catch { - Write-Log "Failed to restore $project, retrying..." + Write-Log -message "Failed to restore $project, retrying..." $retryCount++ if($retryCount -ge $maxTries) { @@ -656,7 +668,7 @@ function Restore-PSPackage continue } - Write-Log "Done restoring $project" + Write-Log -message "Done restoring $project" break } } @@ -2390,10 +2402,10 @@ function Copy-PSGalleryModules Find-DotNet - Restore-PSPackage -ProjectDirs (Split-Path $CsProjPath) -Force:$Force.IsPresent + Restore-PSPackage -ProjectDirs (Split-Path $CsProjPath) -Force:$Force.IsPresent -PSModule $cache = dotnet nuget locals global-packages -l - if ($cache -match "info : global-packages: (.*)") { + if ($cache -match "(?:info : )?global-packages: (.*)") { $nugetCache = $matches[1] } else { @@ -3172,8 +3184,10 @@ function New-NugetConfigFile - + + + <[FEEDNAME]> diff --git a/global.json b/global.json index aab18a8b2ef..d82f13f4dbe 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "3.1.106" + "version": "3.1.405" } } diff --git a/nuget.config b/nuget.config index 5ec994d9118..6548586147e 100644 --- a/nuget.config +++ b/nuget.config @@ -3,9 +3,8 @@ - - - - + + + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index fcdc4f66ada..258681b7161 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -33,8 +33,8 @@ - - + + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index e7879cd90cc..0b2949a7bc3 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -16,7 +16,7 @@ - + @@ -30,7 +30,7 @@ - + diff --git a/src/Modules/PSGalleryModules.csproj b/src/Modules/PSGalleryModules.csproj index 8320faa8111..916db547f16 100644 --- a/src/Modules/PSGalleryModules.csproj +++ b/src/Modules/PSGalleryModules.csproj @@ -1,12 +1,20 @@ - + + PowerShell + Microsoft Corporation + (c) Microsoft Corporation. + + net5.0 + + true + - + - + diff --git a/src/Modules/nuget.config b/src/Modules/nuget.config new file mode 100644 index 00000000000..f5a7f806a36 --- /dev/null +++ b/src/Modules/nuget.config @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs b/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs index 30e9c2b1f38..ea06fd1f2e5 100644 --- a/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs +++ b/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs @@ -239,6 +239,8 @@ public enum XDG_Type DEFAULT } + private static string s_tempHomeDir = null; + /// /// Function for choosing directory location of PowerShell for profile loading. /// @@ -252,7 +254,8 @@ public static string SelectProductNameForDirectory(Platform.XDG_Type dirpath) string envHome = System.Environment.GetEnvironmentVariable(CommonEnvVariableNames.Home); if (envHome == null) { - envHome = GetTemporaryDirectory(); + s_tempHomeDir ??= GetTemporaryDirectory(); + envHome = s_tempHomeDir; } string xdgConfigHomeDefault = Path.Combine(envHome, ".config", "powershell"); diff --git a/src/System.Management.Automation/security/wldpNativeMethods.cs b/src/System.Management.Automation/security/wldpNativeMethods.cs index 08dd920df52..ef19a8817b4 100644 --- a/src/System.Management.Automation/security/wldpNativeMethods.cs +++ b/src/System.Management.Automation/security/wldpNativeMethods.cs @@ -72,28 +72,31 @@ public static SystemEnforcementMode GetSystemLockdownPolicy() /// An EnforcementMode that describes policy. public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle handle) { - // Check the WLDP API - SystemEnforcementMode lockdownPolicy = GetWldpPolicy(path, handle); - if (lockdownPolicy == SystemEnforcementMode.Enforce) + // Check the WLDP File policy via API + var wldpFilePolicy = GetWldpPolicy(path, handle); + if (wldpFilePolicy == SystemEnforcementMode.Enforce) { - return lockdownPolicy; + return wldpFilePolicy; + } + + // Check the AppLocker File policy via API + // This needs to be checked before WLDP audit policy + // So, that we don't end up in Audit mode, + // when we should be enforce mode. + var appLockerFilePolicy = GetAppLockerPolicy(path, handle); + if (appLockerFilePolicy == SystemEnforcementMode.Enforce) + { + return appLockerFilePolicy; } // At this point, LockdownPolicy = Audit or Allowed. // If there was a WLDP policy, but WLDP didn't block it, // then it was explicitly allowed. Therefore, return the result for the file. SystemEnforcementMode systemWldpPolicy = s_cachedWldpSystemPolicy.GetValueOrDefault(SystemEnforcementMode.None); - if ((systemWldpPolicy == SystemEnforcementMode.Enforce) || - (systemWldpPolicy == SystemEnforcementMode.Audit)) - { - return lockdownPolicy; - } - - // Check the AppLocker API - lockdownPolicy = GetAppLockerPolicy(path, handle); - if (lockdownPolicy == SystemEnforcementMode.Enforce) + if ((systemWldpPolicy == SystemEnforcementMode.Audit) || + (systemWldpPolicy == SystemEnforcementMode.Enforce)) { - return lockdownPolicy; + return wldpFilePolicy; } // If there was a system-wide AppLocker policy, but AppLocker didn't block it, @@ -101,7 +104,7 @@ public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle ha if (s_cachedSaferSystemPolicy.GetValueOrDefault(SaferPolicy.Allowed) == SaferPolicy.Disallowed) { - return lockdownPolicy; + return appLockerFilePolicy; } // If it's not set to 'Enforce' by the platform, allow debug overrides diff --git a/src/powershell-unix/powershell-unix.csproj b/src/powershell-unix/powershell-unix.csproj index b815c87dfaf..55f1f86aafb 100644 --- a/src/powershell-unix/powershell-unix.csproj +++ b/src/powershell-unix/powershell-unix.csproj @@ -32,14 +32,15 @@ PreserveNewest PreserveNewest + + en-US\default.help.txt + PreserveNewest + PreserveNewest + - - - - diff --git a/src/powershell-win-core/powershell-win-core.csproj b/src/powershell-win-core/powershell-win-core.csproj index 429d658b1b8..d1cf69b95ef 100644 --- a/src/powershell-win-core/powershell-win-core.csproj +++ b/src/powershell-win-core/powershell-win-core.csproj @@ -49,6 +49,11 @@ preview\pwsh-preview.cmd PreserveNewest + + en-US\default.help.txt + PreserveNewest + PreserveNewest + @@ -59,8 +64,4 @@ - - - - diff --git a/test/hosting/NuGet.Config b/test/hosting/NuGet.Config index c2c6e1b7c2d..765346e5343 100644 --- a/test/hosting/NuGet.Config +++ b/test/hosting/NuGet.Config @@ -2,8 +2,6 @@ - - diff --git a/test/powershell/Installer/WindowsInstaller.Tests.ps1 b/test/powershell/Installer/WindowsInstaller.Tests.ps1 index 5b88d653afe..6167e7fd47e 100644 --- a/test/powershell/Installer/WindowsInstaller.Tests.ps1 +++ b/test/powershell/Installer/WindowsInstaller.Tests.ps1 @@ -1,4 +1,4 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. +# Copyright (c) Microsoft Corporation. # Licensed under the MIT License. Describe "Windows Installer" -Tags "Scenario" { @@ -21,7 +21,8 @@ Describe "Windows Installer" -Tags "Scenario" { ## Running 'Invoke-WebRequest' with WMF download URLs has been failing intermittently, ## because sometimes the URLs lead to a 'this download is no longer available' page. ## We use a retry logic here. Retry for 5 times with 1 second interval. - It "Pre-Requisistes link for '' is reachable: " -TestCases $linkCheckTestCases -skip:$skipTest { + # It "Pre-Requisistes link for '' is reachable: " -TestCases $linkCheckTestCases -Skip:$skipTest { + It "Pre-Requisistes link for '' is reachable: " -TestCases $linkCheckTestCases -Pending { param ($Url) foreach ($i in 1..5) { diff --git a/test/powershell/Modules/Microsoft.PowerShell.Core/Get-Module.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Core/Get-Module.Tests.ps1 index 5d901b7f8d5..82b08aa8439 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Core/Get-Module.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Core/Get-Module.Tests.ps1 @@ -63,6 +63,7 @@ Describe "Get-Module -ListAvailable" -Tags "CI" { ) $env:PSModulePath = Join-Path $testdrive "Modules" + Get-Module Microsoft.PowerShell.Management | Remove-Module -Force } AfterAll { @@ -170,11 +171,17 @@ Describe "Get-Module -ListAvailable" -Tags "CI" { $ModuleVersion ) - $moduleSpecification = @{ModuleName = $name ; ModuleVersion = $ModuleVersion} - $modules = Get-Module -FullyQualifiedName $moduleSpecification - $modules | Should -HaveCount 1 - $modules.Name | Should -BeExactly $ExpectedName - $modules.Version | Should -BeExactly $ModuleVersion + try { + $moduleSpecification = @{ModuleName = $name ; ModuleVersion = $ModuleVersion } + $modules = Get-Module -FullyQualifiedName $moduleSpecification + $modules | Should -HaveCount 1 + $modules.Name | Should -BeExactly $ExpectedName + $modules.Version | Should -BeExactly $ModuleVersion + } catch { + "psmodulepath: $env:PSModulePath" | Write-Verbose -Verbose + $modules | Select-Object Name, Version, path | Out-String -Width 1000 | Write-Verbose -Verbose + throw + } } It "Get-Module -Refresh -ListAvailable" { diff --git a/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 index 838719ad1af..7d91ccb22d4 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 @@ -24,9 +24,9 @@ Describe "SecureString conversion tests" -Tags "CI" { } It "can convert back from a secure string" { - $secret = "abcd" - $ss1 = ConvertTo-SecureString -AsPlainText -Force $secret + $value = "abcd" + $ss1 = ConvertTo-SecureString -AsPlainText -Force $value $ss2 = ConvertFrom-SecureString $ss1 | ConvertTo-SecureString - $ss2 | ConvertFrom-SecureString -AsPlainText | Should -Be $secret + $ss2 | ConvertFrom-SecureString -AsPlainText | Should -Be $value } } diff --git a/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 b/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 index e518ae267c5..f84c83b32ee 100644 --- a/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 +++ b/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 @@ -129,6 +129,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/26" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + $resource = Get-DscResource -Name $name $resource | Should -Not -BeNullOrEmpty $resource.Name | Should -Be $Name @@ -148,6 +152,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/26" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -225,6 +233,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Will only find script from PSDesiredStateConfiguration without modulename" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } @@ -254,6 +266,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/12 and https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -315,6 +331,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -340,6 +360,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -402,6 +426,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if (!$IsLinux) { $result = Invoke-DscResource -Name PSModule -ModuleName $psGetModuleSpecification -Method set -Property @{ Name = 'PsDscResources' @@ -427,6 +455,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + # using create scriptBlock because $using: doesn't work with existing Invoke-DscResource # Verified in Windows PowerShell on 20190814 $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Set -Property @{TestScript = { Write-Output 'test'; return $false }; GetScript = { return @{ } }; SetScript = [scriptblock]::Create("`$global:DSCMachineStatus = $value;return") } @@ -438,6 +470,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Test -Property @{TestScript = { Write-Output 'test'; return $false }; GetScript = { return @{ } }; SetScript = { return } } $result | Should -Not -BeNullOrEmpty @@ -448,6 +483,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Test -Property @{TestScript = { Write-Verbose 'test'; return $true }; GetScript = { return @{ } }; SetScript = { return } } $result | Should -BeTrue -Because "Test method return true" @@ -457,6 +495,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $module = get-module PsDscResources -ListAvailable $moduleSpecification = @{ModuleName = $module.Name; ModuleVersion = $module.Version.ToString() } @@ -520,6 +561,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } { Invoke-DscResource -Name santoheusnaasonteuhsantoheu -Method Test -Property @{TestScript = { Write-Host 'test'; return $true }; GetScript = { return @{ } }; SetScript = { return } } -ErrorAction Stop @@ -531,6 +575,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($IsLinux) { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/12 and https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name PSModule -ModuleName $psGetModuleSpecification -Method Get -Property @{ Name = 'PsDscResources' } $result | Should -Not -BeNullOrEmpty @@ -577,6 +624,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $testString = '890574209347509120348' $result = Invoke-DscResource -Name XmlFileContentResource -ModuleName XmlContentDsc -Property @{Path = $resolvedXmlPath; XPath = '/configuration/appSetting/Test1'; Ensure = 'Present'; Attributes = @{ TestValue2 = $testString; Name = $testString } } -Method Set diff --git a/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 b/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 index c423e8d3392..2a07e05fb45 100644 --- a/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 +++ b/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 @@ -15,6 +15,9 @@ Describe "DSC MOF Compilation" -tags "CI" { if ($IsLinux) { Set-ItResult -Pending -Because "https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOs) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } Write-Verbose "DSC_HOME: ${env:DSC_HOME}" -verbose [Scriptblock]::Create(@" diff --git a/test/powershell/engine/Remoting/PSSession.Tests.ps1 b/test/powershell/engine/Remoting/PSSession.Tests.ps1 index 7d17a7f444d..f507c6ae308 100644 --- a/test/powershell/engine/Remoting/PSSession.Tests.ps1 +++ b/test/powershell/engine/Remoting/PSSession.Tests.ps1 @@ -5,6 +5,11 @@ # PSSession tests for non-Windows platforms # +function GetRandomString() +{ + return [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetRandomFileName()) +} + Describe "New-PSSessionOption parameters for non-Windows platforms" -Tag "CI" { BeforeAll { @@ -36,12 +41,13 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS BeforeAll { $originalDefaultParameterValues = $PSDefaultParameterValues.Clone() - if ($IsWindows) { + # Skip this test for macOS because the latest OS release is incompatible with our shipped libmi for WinRM/OMI. + if ($IsWindows -or $IsMacOS) { $PSDefaultParameterValues['it:skip'] = $true } else { $userName = "User_$(Get-Random -Maximum 99999)" - $userPassword = "Password_$(Get-Random -Maximum 99999)" + $userPassword = GetRandomString $cred = [pscredential]::new($userName, (ConvertTo-SecureString -String $userPassword -AsPlainText -Force)) $soSkipCA = New-PSSessionOption -SkipCACheck $soSkipCN = New-PSSessionOption -SkipCNCheck @@ -54,7 +60,7 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS $testCases = @( @{ - Name = 'Verifies expected error when session options is missing' + Name = 'Verifies expected error when session option is missing' ScriptBlock = { New-PSSession -cn localhost -Credential $cred -Authentication Basic -UseSSL } ExpectedErrorCode = 825 }, diff --git a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 index fd68225c31c..63656b267b1 100644 --- a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 +++ b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 @@ -3,19 +3,25 @@ Import-Module HelpersCommon +function GetRandomString() +{ + return [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetRandomFileName()) +} + Describe "New-PSSession basic test" -Tag @("CI") { It "New-PSSession should not crash powershell" { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - { New-PSSession -ComputerName nonexistcomputer -Authentication Basic } | + { New-PSSession -ComputerName (GetRandomString) -Authentication Basic } | Should -Throw -ErrorId "InvalidOperation,Microsoft.PowerShell.Commands.NewPSSessionCommand" } } @@ -26,13 +32,14 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - $password = ConvertTo-SecureString -String "password" -AsPlainText -Force + $password = ConvertTo-SecureString -String (GetRandomString) -AsPlainText -Force $credential = [PSCredential]::new('username', $password) $err = ({New-PSSession -ComputerName 'localhost' -Credential $credential -Authentication Basic} | Should -Throw -PassThru -ErrorId 'System.Management.Automation.Remoting.PSRemotingDataStructureException,Microsoft.PowerShell.Commands.NewPSSessionCommand') @@ -42,18 +49,20 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { $err.Exception.ErrorCode | Should -Be 801 } - It "New-PSSession should NOT throw a ConnectFailed exception when specifying Basic Auth over HTTPS on Unix" -skip:($IsWindows) { + # Skip this test for macOS because the latest OS release is incompatible with our shipped libmi for WinRM/OMI. + It "New-PSSession should NOT throw a ConnectFailed exception when specifying Basic Auth over HTTPS on Unix" -Skip:($IsWindows) { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - $password = ConvertTo-SecureString -String "password" -AsPlainText -Force + $password = ConvertTo-SecureString -String (GetRandomString) -AsPlainText -Force $credential = [PSCredential]::new('username', $password) # use a Uri that specifies HTTPS to test Basic Auth logic. diff --git a/test/tools/Modules/nuget.config b/test/tools/Modules/nuget.config new file mode 100644 index 00000000000..b0fc73009da --- /dev/null +++ b/test/tools/Modules/nuget.config @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index e428b541c6d..6806d83c74c 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -205,6 +205,9 @@ function Start-PSPackage { # Copy the ThirdPartyNotices.txt so it's part of the package Copy-Item "$RepoRoot/ThirdPartyNotices.txt" -Destination $Source -Force + # Copy the default.help.txt so it's part of the package + Copy-Item "$RepoRoot/assets/default.help.txt" -Destination "$Source/en-US" -Force + # If building a symbols package, we add a zip of the parent to publish if ($IncludeSymbols.IsPresent) { @@ -661,13 +664,14 @@ function Expand-PSSignedBuild # That zip file is used for compliance scan. Remove-Item -Path (Join-Path -Path $buildPath -ChildPath '*.zip') -Recurse - if ($SkipPwshExeCheck) - { - $windowsExecutablePath = (Join-Path $buildPath -ChildPath 'pwsh.dll') - } - else - { - $windowsExecutablePath = (Join-Path $buildPath -ChildPath 'pwsh.exe') + if ($SkipPwshExeCheck) { + $executablePath = (Join-Path $buildPath -ChildPath 'pwsh.dll') + } else { + if ($IsMacOS -or $IsLinux) { + $executablePath = (Join-Path $buildPath -ChildPath 'pwsh') + } else { + $executablePath = (Join-Path $buildPath -ChildPath 'pwsh.exe') + } } Restore-PSModuleToBuild -PublishPath $buildPath @@ -679,12 +683,9 @@ function Expand-PSSignedBuild $options.PSModuleRestore = $true - if (Test-Path -Path $windowsExecutablePath) - { - $options.Output = $windowsExecutablePath - } - else - { + if (Test-Path -Path $executablePath) { + $options.Output = $executablePath + } else { throw 'Could not find pwsh' } @@ -3077,7 +3078,7 @@ function New-MSIXPackage $displayName = $productName - if ($packageName.Contains('-')) { + if ($ProductSemanticVersion.Contains('-')) { $ProductName += 'Preview' $displayName += ' Preview' } @@ -3107,9 +3108,18 @@ function New-MSIXPackage Write-Verbose "Version: $productversion" -Verbose + $isPreview = Test-IsPreview -Version $ProductSemanticVersion + if ($isPreview) { + Write-Verbose "Using Preview assets" -Verbose + } + # Appx manifest needs to be in root of source path, but the embedded version needs to be updated + # cp-459155 is 'CN=Microsoft Windows Store Publisher (Store EKU), O=Microsoft Corporation, L=Redmond, S=Washington, C=US' + # authenticodeFormer is 'CN=Microsoft Corporation, O=Microsoft Corporation, L=Redmond, S=Washington, C=US' + $releasePublisher = 'CN=Microsoft Corporation, O=Microsoft Corporation, L=Redmond, S=Washington, C=US' + $appxManifest = Get-Content "$RepoRoot\assets\AppxManifest.xml" -Raw - $appxManifest = $appxManifest.Replace('$VERSION$', $ProductVersion).Replace('$ARCH$', $Architecture).Replace('$PRODUCTNAME$', $productName).Replace('$DISPLAYNAME$', $displayName) + $appxManifest = $appxManifest.Replace('$VERSION$', $ProductVersion).Replace('$ARCH$', $Architecture).Replace('$PRODUCTNAME$', $productName).Replace('$DISPLAYNAME$', $displayName).Replace('$PUBLISHER$', $releasePublisher) Set-Content -Path "$ProductSourcePath\AppxManifest.xml" -Value $appxManifest -Force # Necessary image assets need to be in source assets folder $assets = @( @@ -3124,11 +3134,6 @@ function New-MSIXPackage $null = New-Item -ItemType Directory -Path "$ProductSourcePath\assets" } - $isPreview = Test-IsPreview -Version $ProductSemanticVersion - if ($isPreview) { - Write-Verbose "Using Preview assets" -Verbose - } - $assets | ForEach-Object { if ($isPreview) { Copy-Item -Path "$RepoRoot\assets\$_-Preview.png" -Destination "$ProductSourcePath\assets\$_.png" diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml index e6f5756b9d3..35a51f65ef2 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml @@ -15,7 +15,7 @@ steps: - powershell: | Import-Module -Force "$(Build.SourcesDirectory)/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1" - SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT) -Destination $(Build.ArtifactStagingDirectory) + SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT2) -Destination $(Build.ArtifactStagingDirectory) displayName: Download packages from PSGallery that need to be updated condition: succeededOrFailed() diff --git a/tools/releaseBuild/azureDevOps/WindowsBuild.yml b/tools/releaseBuild/azureDevOps/WindowsBuild.yml deleted file mode 100644 index 61e3041e338..00000000000 --- a/tools/releaseBuild/azureDevOps/WindowsBuild.yml +++ /dev/null @@ -1,464 +0,0 @@ -jobs: - -- job: BuildJob - displayName: Build - condition: succeeded() - pool: - name: Package ES Standard Build - strategy: - matrix: - Build (x64,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - Build (x86,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x86 - Build (arm,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: arm - Build (arm64,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: arm64 - Build (fxdependent,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: fxdependent - - steps: - - - checkout: self - clean: true - persistCredentials: true - - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - - - powershell: | - Write-Verbose -Verbose "$(Architecture)" - - if ('$(Architecture)' -eq 'fxdependent' -and '$(ReleaseTag)' -match '6.0.*') - { - $vstsCommandString = "vso[task.setvariable variable=SkipFxDependent]true" - } - else - { - $vstsCommandString = "vso[task.setvariable variable=SkipFxDependent]false" - } - - Write-Verbose -Message "$vstsCommandString " -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: 'Skip FxDependent for PS v6.0.*' - - - powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules - - if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' - condition: ne(Variables['SkipFxDependent'], 'true') - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set Version Varibale' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'remove all containers [Port to PSRelease]' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'remove old images [Port to PSRelease]' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - Write-verbose "--git branch ---" -verbose - git branch - exit 0 - displayName: 'Get Environment' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-$(Architecture)-symbols - displayName: 'Build Windows Universal - $(Architecture) Symbols zip' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - if ("$env:Architecture" -eq 'fxdependent') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(Architecture)}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Get Symbols path [Update build.json]' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(Symbols)' - outPathRoot: '$(Symbols)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - New-Item -ItemType Directory -Path $(Symbols)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['Build.Reason'], 'Manual'), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-$(Architecture)-package -BuildPath $(Symbols) -SignedFilesPath $(Symbols)\signed - displayName: 'Build Windows Universal - $(Architecture) Package' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - -- job: ComponentRegistrationJob - displayName: Component Registration - dependsOn: BuildJob - condition: succeeded() - pool: - name: Package ES Standard Build - strategy: - matrix: - release-anycpu: - BuildConfiguration: release - BuildPlatform: any cpu - - steps: - - - powershell: | - ./tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set Version Variable' - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'Remove all containers' - continueOnError: true - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'Remove old images' - continueOnError: true - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - Write-verbose "--git branch ---" -verbose - git branch - exit 0 - displayName: 'Get Environment' - - - powershell: | - ./tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-x64-component-registration - displayName: 'Build Windows Universal - Component Registration' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(componentregistration)' - snapshotForceEnabled: true - -- job: PackageSigningJob - displayName: Package signing - dependsOn: BuildJob - condition: succeeded() - pool: - name: Package ES Standard Build - strategy: - matrix: - release-anycpu: - BuildConfiguration: release - BuildPlatform: any cpu - - steps: - - - powershell: | - tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - continueOnError: true - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $azureVersion = $env:ReleaseTag.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - displayName: 'Set Version Variable' - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifacts' - inputs: - downloadType: specific - continueOnError: true - - - powershell: | - dir "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'dir artifacts directory' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x86.zip" -Destination "$(Build.StagingDirectory)\symbols\x86" - displayName: 'Expand symbols zip - x86' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x64.zip" -Destination "$(Build.StagingDirectory)\symbols\x64" - displayName: 'Expand symbols zip - x64' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-fxdependent.zip" -Destination "$(Build.StagingDirectory)\symbols\fxdependent" - displayName: 'Expand symbols zip - fxdependent' - continueOnError: true - - - powershell: | - tools/releaseBuild/generatePackgeSigning.ps1 -AuthenticodeFiles "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msi","$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msi" -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate Package Signing Xml' - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'print signing xml' - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed msi - x64' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - x64' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed msi - x86' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - x86' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed zip - x64' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - x64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - x86' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - x86' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - arm' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm64' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - arm64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - fxdependent' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - fxdependent' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 803dd01c90f..5e87ff58ec7 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -10,156 +10,162 @@ pr: - master - release* -variables: - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - POWERSHELL_TELEMETRY_OPTOUT: 1 - -# Set AzDevOps Agent to clean the machine after the end of the build resources: -- repo: self - clean: true - -jobs: -- template: templates/checkAzureContainer.yml - -- template: templates/linux.yml - parameters: - buildName: deb - parentJob: DeleteBlob - -- template: templates/linux.yml - parameters: - buildName: rpm - uploadDisplayName: Upload and Sign - parentJob: DeleteBlob - -- template: templates/linux.yml - parameters: - buildName: fxdependent - parentJob: DeleteBlob - -- template: templates/linux.yml - parameters: - buildName: alpine - parentJob: DeleteBlob - -- template: templates/mac.yml - parameters: - parentJob: DeleteBlob - -- template: templates/windows-hosted-build.yml - parameters: - Architecture: x64 - parentJob: DeleteBlob - -- template: templates/windows-hosted-build.yml - parameters: - Architecture: x86 - parentJob: DeleteBlob - -- template: templates/windows-hosted-build.yml - parameters: - Architecture: arm - parentJob: DeleteBlob - -- template: templates/windows-hosted-build.yml - parameters: - Architecture: arm64 - parentJob: DeleteBlob - -- template: templates/windows-hosted-build.yml - parameters: - Architecture: fxdependent - parentJob: DeleteBlob - -- template: templates/windows-hosted-build.yml - parameters: - Architecture: fxdependentWinDesktop - parentJob: DeleteBlob - -- template: templates/windows-packaging.yml - parameters: - Architecture: x64 - parentJob: build_windows_x64 - -- template: templates/windows-packaging.yml - parameters: - Architecture: x86 - parentJob: build_windows_x86 - -- template: templates/windows-packaging.yml - parameters: - Architecture: arm - parentJob: build_windows_arm - -- template: templates/windows-packaging.yml - parameters: - Architecture: arm64 - parentJob: build_windows_arm64 - -- template: templates/windows-packaging.yml - parameters: - Architecture: fxdependent - parentJob: build_windows_fxdependent - -- template: templates/windows-packaging.yml - parameters: - Architecture: fxdependentWinDesktop - parentJob: build_windows_fxdependentWinDesktop - -- template: templates/windows-package-signing.yml - parameters: - parentJobs: - - sign_windows_x64 - - sign_windows_x86 - - sign_windows_arm - - sign_windows_arm64 - - sign_windows_fxdependent - - sign_windows_fxdependentWinDesktop - -- template: templates/mac-package-signing.yml - -- template: templates/compliance.yml - parameters: - parentJobs: - - build_windows_x64 - - build_windows_x86 - - build_windows_fxdependent - - build_windows_fxdependentWinDesktop - -- template: templates/nuget.yml - parameters: - parentJobs: - - WinPackageSigningJob - - upload_deb - - upload_rpm - - upload_alpine - - build_macOS - -- template: templates/json.yml - parameters: - parentJobs: - - WinPackageSigningJob - - upload_deb - - upload_rpm - - upload_alpine - - MacPackageSigningJob - -- template: templates/testartifacts.yml - -- job: release_json - displayName: Create and Upload release.json - pool: - vmImage: 'windows-latest' - steps: - - template: templates/SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - powershell: | - $metadata = Get-Content '$(Build.SourcesDirectory)/tools/metadata.json' -Raw | ConvertFrom-Json - $LTS = $metadata.LTSRelease - @{ ReleaseVersion = "$(Version)"; LTSRelease = $LTS } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json" - Get-Content "$(Build.StagingDirectory)\release.json" - Write-Host "##vso[artifact.upload containerfolder=metadata;artifactname=metadata]$(Build.StagingDirectory)\release.json" - displayName: Create and upload release.json file to build artifact + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + ref: master + +variables: + - name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 + - name: POWERSHELL_TELEMETRY_OPTOUT + value: 1 + +stages: + - stage: prep + jobs: + - template: templates/checkAzureContainer.yml + + - stage: macos + dependsOn: ['prep'] + jobs: + - template: templates/mac.yml + + - template: templates/mac-file-signing.yml + parameters: + parentJob: build_macOS + + - template: templates/mac-package-build.yml + parameters: + parentJob: MacFileSigningJob + + - template: templates/mac-package-signing.yml + parameters: + parentJob: package_macOS + + - stage: linux + dependsOn: ['prep'] + jobs: + - template: templates/linux.yml + parameters: + buildName: deb + + - template: templates/linux.yml + parameters: + buildName: rpm + uploadDisplayName: Upload and Sign + + - template: templates/linux.yml + parameters: + buildName: fxdependent + parentJob: build_deb + + - template: templates/linux.yml + parameters: + buildName: alpine + + - stage: windows + dependsOn: ['prep'] + jobs: + - template: templates/windows-hosted-build.yml + parameters: + Architecture: x64 + + - template: templates/windows-hosted-build.yml + parameters: + Architecture: x86 + + - template: templates/windows-hosted-build.yml + parameters: + Architecture: arm + + - template: templates/windows-hosted-build.yml + parameters: + Architecture: arm64 + + - template: templates/windows-hosted-build.yml + parameters: + Architecture: fxdependent + + - template: templates/windows-hosted-build.yml + parameters: + Architecture: fxdependentWinDesktop + + - template: templates/windows-packaging.yml + parameters: + Architecture: x64 + parentJob: build_windows_x64 + + - template: templates/windows-packaging.yml + parameters: + Architecture: x86 + parentJob: build_windows_x86 + + - template: templates/windows-packaging.yml + parameters: + Architecture: arm + parentJob: build_windows_arm + + - template: templates/windows-packaging.yml + parameters: + Architecture: arm64 + parentJob: build_windows_arm64 + + - template: templates/windows-packaging.yml + parameters: + Architecture: fxdependent + parentJob: build_windows_fxdependent + + - template: templates/windows-packaging.yml + parameters: + Architecture: fxdependentWinDesktop + parentJob: build_windows_fxdependentWinDesktop + + - template: templates/windows-package-signing.yml + parameters: + parentJobs: + - sign_windows_x64 + - sign_windows_x86 + - sign_windows_arm + - sign_windows_arm64 + - sign_windows_fxdependent + - sign_windows_fxdependentWinDesktop + + - stage: compliance + dependsOn: ['windows'] + jobs: + - template: templates/compliance.yml + + - stage: nuget_and_json + dependsOn: ['windows','linux','macOS'] + jobs: + - template: templates/nuget.yml + + - template: templates/json.yml + + - stage: test_and_release_artifacts + dependsOn: ['prep'] + jobs: + - template: templates/testartifacts.yml + + - job: release_json + displayName: Create and Upload release.json + pool: + vmImage: 'windows-latest' + steps: + - checkout: self + clean: true + - template: templates/SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - powershell: | + $metadata = Get-Content '$(Build.SourcesDirectory)/tools/metadata.json' -Raw | ConvertFrom-Json + $LTS = $metadata.LTSRelease + @{ ReleaseVersion = "$(Version)"; LTSRelease = $LTS } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json" + Get-Content "$(Build.StagingDirectory)\release.json" + Write-Host "##vso[artifact.upload containerfolder=metadata;artifactname=metadata]$(Build.StagingDirectory)\release.json" + displayName: Create and upload release.json file to build artifact diff --git a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml index ec688f8a91c..0b8e5f42417 100644 --- a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml +++ b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml @@ -4,9 +4,29 @@ parameters: CreateJson: 'no' steps: +- powershell: | + $path = "./build.psm1" + + if(Test-Path -Path $path) + { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]." + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } + else{ + $path = "./PowerShell/build.psm1" + if(Test-Path -Path $path) + { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]./PowerShell" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } + } + displayName: 'Set repo Root' + - powershell: | $createJson = ("${{ parameters.ReleaseTagVarName }}" -ne "no") - $releaseTag = tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson + $releaseTag = & "$env:REPOROOT/tools/releaseBuild/setReleaseTag.ps1" -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson $version = $releaseTag.Substring(1) $vstsCommandString = "vso[task.setvariable variable=Version]$version" Write-Host ("sending " + $vstsCommandString) diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index 1e8341c8258..65a89a8a79e 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -1,9 +1,13 @@ jobs: - job: DeleteBlob + variables: + runCodesignValidationInjection: false displayName: Delete blob is exists pool: vmImage: windows-latest steps: + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml new file mode 100644 index 00000000000..0a52f887671 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml @@ -0,0 +1,16 @@ +steps: + - powershell: | + $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar + $nativePath = "${dirSeparatorChar}PowerShell" + Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" + + if ((Test-Path "$nativePath")) { + Remove-Item -Path "$nativePath" -Force -Recurse -Verbose -ErrorAction ignore + } + else { + Write-Verbose -Verbose -Message "No cleanup required." + } + + git clone --quiet $env:REPOROOT $nativePath + displayName: Clone PowerShell Repo to /PowerShell + errorActionPreference: silentlycontinue diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 2b47711252f..64641fdd6c0 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -3,6 +3,8 @@ parameters: jobs: - job: compliance + variables: + runCodesignValidationInjection: false displayName: Compliance dependsOn: ${{ parameters.parentJobs }} @@ -13,6 +15,9 @@ jobs: timeoutInMinutes: 180 steps: + - checkout: self + clean: true + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml index ae697e70e33..5fdb9f7d977 100644 --- a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml +++ b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml @@ -1,11 +1,25 @@ +parameters: + - name: "repoRoot" + default: $(REPOROOT) + steps: -- powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules +- pwsh: | + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}/src/Modules' - if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config")) + if(-not (Test-Path "${{ parameters.repoRoot }}/src/Modules/nuget.config")) { throw "nuget.config is not created" } displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules' condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], '')) +- pwsh: | + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsPackageFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}' + + if(-not (Test-Path "${{ parameters.repoRoot }}/nuget.config")) + { + throw "nuget.config is not created" + } + displayName: 'Add nuget.config for Azure DevOps feed for packages' + condition: and(succeededOrFailed(), ne(variables['AzDevOpsPackageFeed'], '')) diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index f72f5518a17..b9949200549 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -3,6 +3,8 @@ parameters: jobs: - job: json + variables: + runCodesignValidationInjection: false displayName: Create Json for Blob dependsOn: ${{ parameters.parentJobs }} @@ -15,6 +17,9 @@ jobs: # inputs: # # displayName: '' + - checkout: self + clean: true + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index af8d42c7c0b..0fac7ae4f24 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -7,11 +7,22 @@ jobs: - job: build_${{ parameters.buildName }} displayName: Build ${{ parameters.buildName }} condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-16.04 dependsOn: ${{ parameters.parentJob }} variables: - build: ${{ parameters.buildName }} + - name: runCodesignValidationInjection + value: false + - name: build + value: ${{ parameters.buildName }} + - group: ESRP + steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: SetVersionVariables.yml parameters: @@ -31,16 +42,19 @@ jobs: displayName: 'Skip Alpine or fxdependent for PS v6.0.*' - template: insert-nuget-config-azfeed.yml - + parameters: + repoRoot: $(REPOROOT) - powershell: | - import-module ./build.psm1 + import-module "$env:REPOROOT/build.psm1" Sync-PSTags -AddRemoteIfMissing displayName: SyncTags condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name '$(build)' + $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' + & "$env:REPOROOT/tools/releaseBuild/vstsbuild.ps1" -ReleaseTag $(ReleaseTagVar) -Name '$(build)' + $env:AzDevOpsFeedPAT2 = $null displayName: 'Build and package' condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) @@ -49,10 +63,19 @@ jobs: displayName: ${{ parameters.uploadDisplayName }} ${{ parameters.buildName }} dependsOn: build_${{ parameters.buildName }} condition: succeeded() - pool: Package ES Standard Build + pool: + vmImage: windows-latest variables: - buildName: ${{ parameters.buildName }} + - name: buildName + value: ${{ parameters.buildName }} + - group: ESRP + steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml @@ -83,32 +106,26 @@ jobs: downloadPath: '$(System.ArtifactsDirectory)\rpm' condition: and(eq(variables['buildName'], 'RPM'),succeeded()) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' - - - powershell: | - $authenticodefiles = @() - Get-ChildItem -Path '$(System.ArtifactsDirectory)\rpm\*.rpm' -recurse | ForEach-Object { $authenticodefiles += $_.FullName} - tools/releaseBuild/generatePackgeSigning.ps1 -LinuxFiles $authenticodeFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate RPM Signing Xml' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture RPM signing xml' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - task: PkgESCodeSign@10 - displayName: 'CodeSign RPM $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(System.ArtifactsDirectory) + pattern: | + **\*.rpm + **\*.deb + **\*.tar.gz + + - ${{ if eq(variables['buildName'], 'RPM') }}: + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\rpm + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-450779-Pgp" + pattern: | + **\*.rpm + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - DEB and tar.gz' inputs: @@ -122,6 +139,7 @@ jobs: parameters: artifactPath: $(System.ArtifactsDirectory)\finished\release + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - RPM - Unsigned' inputs: @@ -132,10 +150,11 @@ jobs: ContainerName: '$(AzureVersion)' condition: and(and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - RPM - Signed' inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\*' + SourcePath: '$(Build.StagingDirectory)\signedPackages\release\*' azureSubscription: '$(AzureFileCopySubscription)' Destination: AzureBlob storage: '$(StorageAccount)' @@ -149,7 +168,7 @@ jobs: - template: upload-final-results.yml parameters: - artifactPath: '$(Build.StagingDirectory)\signedPackages' + artifactPath: '$(Build.StagingDirectory)\signedPackages\release' condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml new file mode 100644 index 00000000000..0332ec560d0 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -0,0 +1,104 @@ +parameters: + parentJob: '' + +jobs: + - job: MacFileSigningJob + displayName: macOS File signing + dependsOn: ${{ parameters.parentJob }} + condition: succeeded() + pool: + name: Package ES Standard Build + variables: + - group: ESRP + + steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true + + - template: shouldSign.yml + + - template: SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - task: DownloadBuildArtifacts@0 + inputs: + artifactName: 'macosBinResults' + itemPattern: '**/*.zip' + downloadPath: '$(System.ArtifactsDirectory)\Symbols' + + - pwsh: | + Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse + displayName: 'Capture Downloaded Artifacts' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + + - pwsh: | + $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*.zip' + Write-Verbose -Verbose "Zip Path: $zipPath" + + $expandedFolder = $zipPath.BaseName + Write-Host "sending.. vso[task.setvariable variable=SymbolsFolder]$expandedFolder" + Write-Host "##vso[task.setvariable variable=SymbolsFolder]$expandedFolder" + + Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force + displayName: Expand symbols zip + + - pwsh: | + Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse + displayName: 'Capture artifacts dir Binaries' + + - pwsh: | + Get-ChildItem "$(System.ArtifactsDirectory)\$(SymbolsFolder)" -Recurse -Include pwsh, *.dylib + displayName: 'Capture Expanded Binaries' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + + - pwsh: | + $null = new-item -type directory -path "$(Build.StagingDirectory)\macos" + $zipFile = "$(Build.StagingDirectory)\macos\powershell-files-$(Version)-osx-x64.zip" + Get-ChildItem "$(System.ArtifactsDirectory)\$(SymbolsFolder)" -Recurse -Include pwsh, *.dylib | + Compress-Archive -Destination $zipFile + Write-Host $zipFile + displayName: 'Compress macOS binary files' + + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(Build.StagingDirectory)\macos + signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages + certificateId: "CP-401337-Apple" + pattern: | + **\*.zip + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + + - pwsh: | + $destination = "$(System.ArtifactsDirectory)\azureMacOs" + New-Item -Path $destination -Type Directory + $zipPath = Get-ChildItem "$(Build.StagingDirectory)\signedMacOSPackages\powershell-*.zip" -Recurse | select-object -expandproperty fullname + foreach ($z in $zipPath) { Expand-Archive -Path $z -DestinationPath $destination } + displayName: 'Extract and copy macOS artifacts for upload' + condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + + - template: upload-final-results.yml + parameters: + artifactPath: $(System.ArtifactsDirectory)\azureMacOs + artifactFilter: "*" + artifactName: signedMacOsBins + condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + + - ${{ if eq(variables['SHOULD_SIGN'], 'true') }}: + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(System.ArtifactsDirectory)\azureMacOs + pattern: | + **\* + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)' + snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml new file mode 100644 index 00000000000..8e89618c1c2 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -0,0 +1,109 @@ +parameters: + parentJob: '' + jobName: 'package_macOS' + +jobs: +- job: ${{ parameters.jobName }} + displayName: Package macOS + dependsOn: ${{ parameters.parentJob }} + condition: succeeded() + pool: Hosted Mac Internal + variables: + # Turn off Homebrew analytics + HOMEBREW_NO_ANALYTICS: 1 + runCodesignValidationInjection: false + steps: + - pwsh: | + # create folder + sudo mkdir /PowerShell + + # make the current user the owner + sudo chown $env:USER /PowerShell + displayName: 'Create /PowerShell' + + - template: shouldSign.yml + + #- task: @ + # inputs: + # + # displayName: '' + - template: SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) + + - template: cloneToOfficialPath.yml + + - task: DownloadBuildArtifacts@0 + displayName: Download macosBinResults + inputs: + artifactName: 'macosBinResults' + itemPattern: '**/*.zip' + downloadPath: '$(System.ArtifactsDirectory)/Symbols' + + - task: DownloadBuildArtifacts@0 + displayName: Download signedMacOsBins + inputs: + artifactName: 'signedMacOsBins' + itemPattern: '**/*' + downloadPath: '$(System.ArtifactsDirectory)/macOsBins' + condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + + - pwsh: | + Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse + displayName: 'Capture Downloaded Artifacts' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + + - pwsh: | + $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*.zip' + Write-Verbose -Verbose "Zip Path: $zipPath" + + $expandedFolder = $zipPath.BaseName + Write-Host "sending.. vso[task.setvariable variable=SymbolsFolder]$expandedFolder" + Write-Host "##vso[task.setvariable variable=SymbolsFolder]$expandedFolder" + + Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force + displayName: Expand symbols zip + + - pwsh: | + Import-Module $(PowerShellRoot)/build.psm1 -Force + Import-Module $(PowerShellRoot)/tools/packaging -Force + $signedFilesPath = '$(System.ArtifactsDirectory)/macOsBins/signedMacOsBins/' + $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' + + Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath + displayName: Merge signed files with Build + condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + + - pwsh: | + Import-Module $(PowerShellRoot)/build.psm1 -Force + Import-Module $(PowerShellRoot)/tools/packaging -Force + + $destFolder = '$(System.ArtifactsDirectory)\signedZip' + $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' + + $null = New-Item -ItemType Directory -Path $destFolder -Force + + $BuildPackagePath = New-PSBuildZip -BuildPath $BuildPath -DestinationFolder $destFolder + + Write-Verbose -Verbose "New-PSSignedBuildZip returned `$BuildPackagePath as: $BuildPackagePath" + Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$BuildPackagePath" + + $vstsCommandString = "vso[task.setvariable variable=BuildPackagePath]$BuildPackagePath" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Compress signed files + + - pwsh: | + tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap + displayName: 'Bootstrap VM' + + - pwsh: | + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" + displayName: 'Package' + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)' + snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index c693c78efa8..8d13693ca5a 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -1,15 +1,22 @@ +parameters: + parentJob: '' + jobs: - job: MacPackageSigningJob displayName: macOS Package signing - dependsOn: build_macOS + dependsOn: ${{ parameters.parentJob }} condition: succeeded() pool: name: Package ES Standard Build variables: - BuildConfiguration: release - BuildPlatform: any cpu + - group: ESRP steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml @@ -18,26 +25,24 @@ jobs: ReleaseTagVar: $(ReleaseTagVar) - task: DownloadBuildArtifacts@0 - displayName: 'Download artifacts' inputs: - downloadType: specific - itemPattern: | - **/*.tar.gz - **/*.pkg + artifactName: 'macosPkgResults' + itemPattern: '**/*' + downloadPath: '$(System.ArtifactsDirectory)' - - powershell: | + - pwsh: | dir "$(System.ArtifactsDirectory)\*" -Recurse displayName: 'Capture Downloaded Artifacts' # Diagnostics is not critical it passes every time it runs continueOnError: true - - powershell: | + - pwsh: | $null = new-item -type directory -path "$(Build.StagingDirectory)\macos" $zipFile = "$(Build.StagingDirectory)\macos\powershell-$(Version)-osx-x64.zip" - Compress-Archive -Path "$(System.ArtifactsDirectory)\results\powershell-$(Version)-osx-x64.pkg" -Destination $zipFile + Compress-Archive -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-x64.pkg" -Destination $zipFile Write-Host $zipFile - $ltsPkgPath = "$(System.ArtifactsDirectory)\results\powershell-lts-$(Version)-osx-x64.pkg" + $ltsPkgPath = "$(System.ArtifactsDirectory)\macosPkgResults\powershell-lts-$(Version)-osx-x64.pkg" if(Test-Path $ltsPkgPath) { @@ -47,34 +52,22 @@ jobs: } displayName: 'Compress macOS Package' - - powershell: | - $pkgFiles = "$(Build.StagingDirectory)\macos\powershell-$(Version)-osx-x64.zip", "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-x64.zip" - tools/releaseBuild/generatePackgeSigning.ps1 -MacDeveloperFiles $pkgFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate macOS Package Signing Xml' - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture macOS signing xml' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedMacOSPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(Build.StagingDirectory)\macos + signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages + certificateId: "CP-401337-Apple" + pattern: | + **\*.zip + useMinimatch: true + shouldSign: $(SHOULD_SIGN) - template: upload-final-results.yml parameters: - artifactPath: $(System.ArtifactsDirectory)\results + artifactPath: $(System.ArtifactsDirectory)\macosPkgResults artifactFilter: "*.tar.gz" - - powershell: | + - pwsh: | $destination = "$(System.ArtifactsDirectory)\azureMacOs" New-Item -Path $destination -Type Directory $zipPath = dir "$(Build.StagingDirectory)\signedMacOSPackages\powershell-*.zip" -Recurse | select-object -expandproperty fullname @@ -90,10 +83,10 @@ jobs: artifactFilter: "*.pkg" condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - powershell: | + - pwsh: | $null = new-item -type directory -path "$(Build.StagingDirectory)\macos-unsigned" - Copy-Item -Path "$(System.ArtifactsDirectory)\results\powershell-$(Version)-osx-x64.pkg" -Destination "$(Build.StagingDirectory)\macos-unsigned" - Copy-Item -Path "$(System.ArtifactsDirectory)\results\powershell-$(Version)-osx-x64.tar.gz" -Destination "$(Build.StagingDirectory)\macos-unsigned" + Copy-Item -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-x64.pkg" -Destination "$(Build.StagingDirectory)\macos-unsigned" + Copy-Item -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-x64.tar.gz" -Destination "$(Build.StagingDirectory)\macos-unsigned" displayName: 'Create unsigned folder to upload' condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) @@ -117,9 +110,6 @@ jobs: ContainerName: '$(AzureVersion)' condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index e586ab2829a..b8425cd2ace 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -1,13 +1,11 @@ parameters: jobName: 'build_macOS' - parentJob: '' jobs: - job: ${{ parameters.jobName }} displayName: Build macOS condition: succeeded() pool: Hosted Mac Internal - dependsOn: ${{ parameters.parentJob }} variables: # Turn off Homebrew analytics HOMEBREW_NO_ANALYTICS: 1 @@ -20,7 +18,7 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) - - powershell: | + - pwsh: | # create folder sudo mkdir /PowerShell @@ -28,30 +26,21 @@ jobs: sudo chown $env:USER /PowerShell displayName: 'Create /PowerShell' - - powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell + - template: cloneToOfficialPath.yml - - powershell: | + - pwsh: | tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap displayName: 'Bootstrap VM' - - powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)/src/Modules" - - if(-not (Test-Path "$(PowerShellRoot)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' - - - powershell: | - $env:AZDEVOPSFEEDPAT = '$(AzDevOpsFeedPAT)' - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -ExtraPackage "tar" -location $(PowerShellRoot) -Build - $env:AZDEVOPSFEEDPAT = $null - displayName: 'Build and Package' + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + parameters: + repoRoot: $(PowerShellRoot) + + - pwsh: | + $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults + $env:AzDevOpsFeedPAT2 = $null + displayName: 'Build' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index ca1de4f8d0f..3a4612d119e 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -12,16 +12,35 @@ jobs: timeoutInMinutes: 90 variables: - GenAPIToolPath: '$(System.ArtifactsDirectory)/GenAPI' - PackagePath: '$(System.ArtifactsDirectory)/UnifiedPackagePath' - winFxdPath: '$(System.ArtifactsDirectory)/winFxd' - winFxdWinDesktopPath: '$(System.ArtifactsDirectory)/winFxdWinDesktop' - linuxFxdPath: '$(System.ArtifactsDirectory)/linuxFxd' + - name: runCodesignValidationInjection + value: false + - name: build + value: ${{ parameters.buildName }} + - group: ESRP + - name: GenAPIToolPath + value: '$(System.ArtifactsDirectory)/GenAPI' + - name: PackagePath + value: '$(System.ArtifactsDirectory)/UnifiedPackagePath' + - name: winFxdPath + value: '$(System.ArtifactsDirectory)/winFxd' + - name: winFxdWinDesktopPath + value: '$(System.ArtifactsDirectory)/winFxdWinDesktop' + - name: linuxFxdPath + value: '$(System.ArtifactsDirectory)/linuxFxd' steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true + + - template: SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) - powershell: | - $content = Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json + $content = Get-Content "$env:REPOROOT/global.json" -Raw | ConvertFrom-Json $vstsCommandString = "vso[task.setvariable variable=SDKVersion]$($content.sdk.version)" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" @@ -89,8 +108,9 @@ jobs: versionSpec: 4.9.3 - powershell: | - Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 - Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging + Import-Module $env:REPOROOT\build.psm1 + Import-Module $env:REPOROOT\tools\packaging + Find-Dotnet New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinFxdBinPath '$(winFxdPath)' -LinuxFxdBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" displayName: 'Create Nuget Package Folders' @@ -101,9 +121,9 @@ jobs: displayName: Capture fxd folders - powershell: | - Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 - Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging - + Import-Module $env:REPOROOT\build.psm1 + Import-Module $env:REPOROOT\tools\packaging + Find-Dotnet # Create unified package first New-GlobalToolNupkg -UnifiedPackage -LinuxBinPath "$(linuxFxdPath)" -WindowsBinPath "$(winFxdPath)" -WindowsDesktopBinPath "$(winFxdWinDesktopPath)" -PackageVersion "$(Version)" -DestinationPath "$(PackagePath)\globaltool" @@ -115,31 +135,25 @@ jobs: Get-ChildItem "$(PackagePath)" -Recurse displayName: Capture generated packages - - powershell: | - $packages = Get-ChildItem "$(PackagePath)\*.nupkg", "$(PackagePath)\globaltool\*.nupkg" | Select-Object -ExpandProperty FullName - - if($packages.Count -lt 1) - { - throw "No packages created" - } - - $(Build.SourcesDirectory)\tools\releaseBuild\generatePackgeSigning.ps1 -Path $(PackagePath)\NugetSigning.xml -NuPkgFiles $packages - displayName: Create signing file - - - task: PkgESCodeSign@10 - displayName: 'CodeSign Nuget Packages' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(PackagePath)\NugetSigning.xml' - inPathRoot: '$(PackagePath)' - outPathRoot: '$(System.ArtifactsDirectory)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(PackagePath) + signOutputPath: $(System.ArtifactsDirectory)\signed + certificateId: "CP-401405" + pattern: | + **\*.nupkg + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + + - pwsh: | + if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } + Copy-Item -Path '$(PackagePath)\*.nupkg' -Destination '$(System.ArtifactsDirectory)\signed\' -Verbose -Force + Copy-Item -Path '$(PackagePath)\globaltool\*.nupkg' -Destination '$(System.ArtifactsDirectory)\signed\' -Verbose -Force + displayName: Fake copy when not signing + condition: eq(variables['SHOULD_SIGN'], 'false') - powershell: | - Import-Module $(Build.SourcesDirectory)\build.psm1 -Force + Import-Module "${env:REPOROOT}\build.psm1" -Force Get-ChildItem -Recurse "$(System.ArtifactsDirectory)\signed\*.nupkg" -Verbose | ForEach-Object { Start-NativeExecution -sb { nuget.exe verify -All $_.FullName } } displayName: Verify all packages are signed @@ -157,7 +171,11 @@ jobs: artifactPath: '$(System.ArtifactsDirectory)\signed' - powershell: | - $null = New-Item -Path "$(System.ArtifactsDirectory)\signed\globaltool" -ItemType Directory + if (-not (Test-Path "$(System.ArtifactsDirectory)\signed\globaltool")) + { + $null = New-Item -Path "$(System.ArtifactsDirectory)\signed\globaltool" -ItemType Directory -Force + } + Move-Item -Path "$(System.ArtifactsDirectory)\signed\PowerShell.*" -Destination "$(System.ArtifactsDirectory)\signed\globaltool" -Force Get-ChildItem "$(System.ArtifactsDirectory)\signed\globaltool" -Recurse displayName: Move global tool packages to subfolder and capture diff --git a/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml b/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml index 287ca50ba5f..341a44d02b9 100644 --- a/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml +++ b/tools/releaseBuild/azureDevOps/templates/upload-final-results.yml @@ -2,13 +2,14 @@ parameters: artifactPath: artifactFilter: '*' condition: succeeded() + artifactName: finalResults steps: - powershell: | Get-ChildItem -Path '${{ parameters.artifactPath }}' -File -filter '${{ parameters.artifactFilter }}' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty FullName | ForEach-Object { - Write-Host "##vso[artifact.upload containerfolder=finalResults;artifactname=finalResults]$_" + Write-Host "##vso[artifact.upload containerfolder=${{ parameters.artifactName }};artifactname=${{ parameters.artifactName }}]$_" } - displayName: Upload Final Artifacts ${{ parameters.artifactFilter }} from ${{ parameters.artifactPath }} + displayName: Upload ${{ parameters.artifactName }} Artifacts ${{ parameters.artifactFilter }} from ${{ parameters.artifactPath }} condition: ${{ parameters.condition }} diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index 97d3bea8d1d..c7ef645a878 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -7,6 +7,8 @@ jobs: condition: succeeded() pool: Package ES Standard Build steps: + - checkout: self + clean: true - template: ./SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-build.yml deleted file mode 100644 index 8aab17585f3..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/windows-build.yml +++ /dev/null @@ -1,126 +0,0 @@ -parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - -jobs: -- job: build_windows_${{ parameters.Architecture }} - displayName: Build Windows - ${{ parameters.Architecture }} - condition: succeeded() - pool: - name: Package ES Standard Build - variables: - BuildConfiguration: ${{ parameters.BuildConfiguration }} - BuildPlatform: ${{ parameters.BuildPlatform }} - Architecture: ${{ parameters.Architecture }} - - steps: - - - checkout: self - clean: true - persistCredentials: true - - - template: shouldSign.yml - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: insert-nuget-config-azfeed.yml - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'Remove all containers [Port to PSRelease]' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'Remove old images [Port to PSRelease]' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - exit 0 - displayName: 'Capture docker info' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-$(Architecture)-symbols - displayName: 'Build Windows Universal - $(Architecture) Symbols zip' - - - powershell: | - if ("$env:Architecture" -like 'fxdependent*') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - - - powershell: | - $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(Architecture)}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Get Symbols path [Update build.json]' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(Symbols)' - outPathRoot: '$(Symbols)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - powershell: | - New-Item -ItemType Directory -Path $(Symbols)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-$(Architecture)-package -BuildPath $(Symbols) -SignedFilesPath $(Symbols)\signed - displayName: 'Build Windows Universal - $(Architecture) Package' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 412a5d13f27..55d92c5cc91 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -1,8 +1,13 @@ parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - parentJob: '' + - name: BuildConfiguration + default: release + - name: BuildPlatform + default: any cpu + - name: Architecture + default: x64 + - name: parentJob + default: '' + jobs: - job: build_windows_${{ parameters.Architecture }} displayName: Build Windows - ${{ parameters.Architecture }} @@ -26,13 +31,11 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) - - template: insert-nuget-config-azfeed.yml + - template: cloneToOfficialPath.yml - - powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - $null = New-Item -ItemType Directory -Path /Powershell -Force - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + parameters: + repoRoot: $(PowerShellRoot) - powershell: | diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 966173ddba6..c715891dcde 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -8,12 +8,18 @@ jobs: ${{ parameters.parentJobs }} condition: succeeded() pool: - name: Package ES Standard Build + vmImage: windows-latest variables: - BuildConfiguration: release - BuildPlatform: any cpu + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: ESRP steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml - template: SetVersionVariables.yml @@ -34,32 +40,16 @@ jobs: # Diagnostics is not critical it passes every time it runs continueOnError: true - - powershell: | - $authenticodefiles = @( - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msi" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msi" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.msix" - ) - tools/releaseBuild/generatePackgeSigning.ps1 -AuthenticodeFiles $authenticodeFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate Package Signing Xml' - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture signing xml' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\signed + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-230012" + pattern: | + **\*.msi + **\*.msix + useMinimatch: true + shouldSign: $(SHOULD_SIGN) - powershell: | new-item -itemtype Directory -path '$(Build.StagingDirectory)\signedPackages' @@ -103,8 +93,13 @@ jobs: msi: no msix: no - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(Build.StagingDirectory) + pattern: | + **\*.msix + **\*.msi + **\*.zip - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 457c26e6881..a2428f80cb1 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -1,8 +1,12 @@ parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - parentJob: '' + - name: BuildConfiguration + default: release + - name: BuildPlatform + default: any cpu + - name: Architecture + default: x64 + - name: parentJob + default: '' jobs: - job: sign_windows_${{ parameters.Architecture }} @@ -12,10 +16,15 @@ jobs: pool: name: Package ES Standard Build variables: - BuildConfiguration: ${{ parameters.BuildConfiguration }} - BuildPlatform: ${{ parameters.BuildPlatform }} - Architecture: ${{ parameters.Architecture }} - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: BuildPlatform + value: ${{ parameters.BuildPlatform }} + - name: Architecture + value: ${{ parameters.Architecture }} + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: ESRP steps: @@ -23,24 +32,14 @@ jobs: clean: true persistCredentials: true + - checkout: ComplianceRepo + clean: true + - template: shouldSign.yml - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - powershell: | $pkgFilter = if ( '$(Architecture)' -eq 'arm' ) { "arm32" @@ -60,22 +59,9 @@ jobs: itemPattern: '**/*$(PkgFilter).zip' downloadPath: '$(System.ArtifactsDirectory)\Symbols' - - powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" + - template: cloneToOfficialPath.yml - if ((Test-Path "\PowerShell")) { - Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose - } - else { - Write-Verbose -Verbose -Message "No cleanup required." - } - - git clone --quiet $env:BUILD_REPOSITORY_LOCALPATH '\PowerShell' - - displayName: Clone PowerShell Repo to /PowerShell - errorActionPreference: silentlycontinue - - - powershell: | + - pwsh: | # cleanup previous install if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force @@ -83,7 +69,7 @@ jobs: $toolsDir = New-Item -ItemType Directory -Path '$(Build.ArtifactStagingDirectory)\tools' $wixUri = 'https://github.com/wixtoolset/wix3/releases/download/wix311rtm/wix311-binaries.zip' - Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' + Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' -MaximumRetryCount 5 -RetryIntervalSec 10 Import-Module '$(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1' Install-WixZip -zipPath '$(Build.ArtifactStagingDirectory)\tools\wix.zip' @@ -105,33 +91,75 @@ jobs: Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force displayName: Expand symbols zip - - powershell: | - if ("$env:Architecture" -like 'fxdependent*') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - outPathRoot: '$(System.ArtifactsDirectory)\signed' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - pwsh: | + $fullSymbolsFolder = "$(System.ArtifactsDirectory)\$($env:SYMBOLSFOLDER)" + + $filesToSignDirectory = "$(System.ArtifactsDirectory)\toBeSigned" + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $signedFilesDirectory = "$(System.ArtifactsDirectory)\signed" + $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force + + $itemsToCopyWithRecurse = @( + "$($fullSymbolsFolder)\*.ps1" + "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" + ) + + $itemsToCopy = @{ + "$($fullSymbolsFolder)\*.ps1" = "" + "$($fullSymbolsFolder)\Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" + "$($fullSymbolsFolder)\Microsoft.WSMan.*.dll" = "" + "$($fullSymbolsFolder)\Modules\CimCmdlets\CimCmdlets.psd1" = "Modules\CimCmdlets" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Diagnostics.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Event.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\GetEvent.types.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Microsoft.PowerShell.Diagnostics.psd1" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Host\Microsoft.PowerShell.Host.psd1" = "Modules\Microsoft.PowerShell.Host" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Management\Microsoft.PowerShell.Management.psd1" = "Modules\Microsoft.PowerShell.Management" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Microsoft.PowerShell.Security.psd1" = "Modules\Microsoft.PowerShell.Security" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Utility\Microsoft.PowerShell.Utility.psd1" = "Modules\Microsoft.PowerShell.Utility" + "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\Microsoft.WSMan.Management.psd1" = "Modules\Microsoft.WSMan.Management" + "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\WSMan.format.ps1xml" = "Modules\Microsoft.WSMan.Management" + "$($fullSymbolsFolder)\Modules\PSDiagnostics\PSDiagnostics.ps?1" = "Modules\PSDiagnostics" + "$($fullSymbolsFolder)\pwsh.dll" = "" + "$($fullSymbolsFolder)\System.Management.Automation.dll" = "" + } - - powershell: | - New-Item -ItemType Directory -Path $(System.ArtifactsDirectory)\signedZip -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) + $itemsToExclude = @( + ) - - powershell: | + if ("$env:Architecture" -notlike 'fxdependent*') + { + $itemsToCopy += @{"$($fullSymbolsFolder)\pwsh.exe" = ""} + } + + Write-Verbose -verbose "recusively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" + Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude + + foreach($pattern in $itemsToCopy.Keys) { + $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern + $null = New-Item -ItemType Directory -Path $destinationFolder -Force + Write-Verbose -verbose "copying $pattern to $destinationFolder" + Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose + } + displayName: 'Prepare files to be signed' + + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\toBeSigned + signOutputPath: $(System.ArtifactsDirectory)\signed + certificateId: "CP-230012" + pattern: | + **\*.dll + **\*.psd1 + **\*.psm1 + **\*.ps1xml + **\*.ps1 + **\*.exe + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force $signedFilesPath = '$(System.ArtifactsDirectory)\signed\' @@ -141,29 +169,41 @@ jobs: $dlls = Get-ChildItem $BuildPath\*.dll -Recurse $signatures = $dlls | Get-AuthenticodeSignature $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned'}| select-object -ExpandProperty Path - tools/releaseBuild/generatePackgeSigning.ps1 -ThirdPartyFiles $missingSignatures -path "$(System.ArtifactsDirectory)\thirtdparty.xml" - displayName: Create ThirdParty Signing Xml - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: PkgESCodeSign@10 - displayName: 'CodeSign ThirdParty $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\thirtdparty.xml' - inPathRoot: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - outPathRoot: '$(System.ArtifactsDirectory)\signedThirdParty' + Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" + + $filesToSignDirectory = "$(System.ArtifactsDirectory)\thirdPartyToBeSigned" + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $signedFilesDirectory = "$(System.ArtifactsDirectory)\thirdPartySigned" + $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force + + $missingSignatures | ForEach-Object { + Copy-Item -Path $_ -Destination $filesToSignDirectory + } + + displayName: Create ThirdParty Signing Folder condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\thirdPartyToBeSigned + signOutputPath: $(System.ArtifactsDirectory)\thirdPartySigned + certificateId: "CP-231522" + pattern: | + **\*.dll + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + - powershell: | - Get-ChildItem '$(System.ArtifactsDirectory)\signedThirdParty\*' + Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' displayName: Captrue ThirdParty Signed files condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - powershell: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\signedThirdParty\' + $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 48b9e833844..70091eac8b5 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -17,11 +17,6 @@ variables: value: 1 - group: Azure Blob variable group -# Set AzDevOps Agent to clean the machine after the end of the build -resources: -- repo: self - clean: true - jobs: - job: rename displayName: Name the build @@ -29,6 +24,8 @@ jobs: pool: vmImage: 'windows-latest' steps: + - checkout: self + clean: true - template: ./templates/SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/generatePackgeSigning.ps1 b/tools/releaseBuild/generatePackgeSigning.ps1 index 083bb6b7ad5..e77788cfa97 100644 --- a/tools/releaseBuild/generatePackgeSigning.ps1 +++ b/tools/releaseBuild/generatePackgeSigning.ps1 @@ -8,7 +8,10 @@ param( [string[]] $NuPkgFiles, [string[]] $MacDeveloperFiles, [string[]] $LinuxFiles, - [string[]] $ThirdPartyFiles + [string[]] $ThirdPartyFiles, + [string[]] $MsixFiles, + [ValidateSet('release','preview')] + [string] $MsixCertType = 'preview' ) if ((!$AuthenticodeDualFiles -or $AuthenticodeDualFiles.Count -eq 0) -and @@ -16,6 +19,7 @@ if ((!$AuthenticodeDualFiles -or $AuthenticodeDualFiles.Count -eq 0) -and (!$NuPkgFiles -or $NuPkgFiles.Count -eq 0) -and (!$MacDeveloperFiles -or $MacDeveloperFiles.Count -eq 0) -and (!$LinuxFiles -or $LinuxFiles.Count -eq 0) -and + (!$MsixFiles -or $MsixFiles.Count -eq 0) -and (!$ThirdPartyFiles -or $ThirdPartyFiles.Count -eq 0)) { throw "At least one file must be specified" @@ -95,6 +99,14 @@ foreach ($file in $ThirdPartyFiles) { New-FileElement -File $file -SignType 'ThirdParty' -XmlDoc $signingXml -Job $job } +foreach ($file in $MsixFiles) { + # 'CP-459155' is supposed to work for the store + # AuthenticodeFormer works for sideloading and via a workaround, through the store + # ---------------------------------------------- + # update releasePublisher in packaging.psm1 when this is changed + New-FileElement -File $file -SignType 'AuthenticodeFormer' -XmlDoc $signingXml -Job $job +} + $signingXml.Save($path) $updateScriptPath = Join-Path -Path $PSScriptRoot -ChildPath 'updateSigning.ps1' & $updateScriptPath -SigningXmlPath $path diff --git a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 index be9ade24519..e2d6735bd4d 100644 --- a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 +++ b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 @@ -1,4 +1,4 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. +# Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # PowerShell Script to build and package PowerShell from specified form and branch @@ -10,14 +10,20 @@ param ( [string] $location = $env:BUILD_REPOSITORY_LOCALPATH, # Destination location of the package on docker host + [Parameter(Mandatory, ParameterSetName = 'packageSigned')] + [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] [Parameter(Mandatory, ParameterSetName = 'Build')] [string] $destination = '/mnt', + [Parameter(Mandatory, ParameterSetName = 'packageSigned')] + [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] [Parameter(Mandatory, ParameterSetName = 'Build')] [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d+)?)?$")] [ValidateNotNullOrEmpty()] [string]$ReleaseTag, + [Parameter(ParameterSetName = 'packageSigned')] + [Parameter(ParameterSetName = 'IncludeSymbols')] [Parameter(ParameterSetName = 'Build')] [ValidateSet("zip", "tar")] [string[]]$ExtraPackage, @@ -25,13 +31,23 @@ param ( [Parameter(Mandatory, ParameterSetName = 'Bootstrap')] [switch] $BootStrap, + [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] [Parameter(Mandatory, ParameterSetName = 'Build')] - [switch] $Build + [switch] $Build, + + [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] + [switch] $Symbols, + + [Parameter(Mandatory, ParameterSetName = 'packageSigned')] + [ValidatePattern("-signed.zip$")] + [string]$BuildZip, + + [string]$ArtifactName = 'result' ) $repoRoot = $location -if ($Build.IsPresent) { +if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { $releaseTagParam = @{ } if ($ReleaseTag) { $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } @@ -59,8 +75,11 @@ try { Start-PSBootstrap -Package } - if ($Build.IsPresent) { - Start-PSBuild -Configuration 'Release' -Crossgen -PSModuleRestore @releaseTagParam + if ($PSCmdlet.ParameterSetName -eq 'packageSigned') { + Write-Verbose "Expanding signed build $BuildZip ..." -Verbose + Expand-PSSignedBuild -BuildZip $BuildZip + + Remove-Item -Path $BuildZip Start-PSPackage @releaseTagParam switch ($ExtraPackage) { @@ -74,19 +93,42 @@ try { } } } + + if ($Build.IsPresent) { + if ($Symbols.IsPresent) { + Start-PSBuild -Configuration 'Release' -Crossgen -NoPSModuleRestore @releaseTagParam + $pspackageParams = @{} + $pspackageParams['Type']='zip' + $pspackageParams['IncludeSymbols']=$Symbols.IsPresent + Write-Verbose "Starting powershell packaging(zip)..." -Verbose + Start-PSPackage @pspackageParams @releaseTagParam + } else { + Start-PSBuild -Configuration 'Release' -Crossgen -PSModuleRestore @releaseTagParam + Start-PSPackage @releaseTagParam + switch ($ExtraPackage) { + "tar" { Start-PSPackage -Type tar @releaseTagParam } + } + + if ($LTS) { + Start-PSPackage @releaseTagParam -LTS + switch ($ExtraPackage) { + "tar" { Start-PSPackage -Type tar @releaseTagParam -LTS } + } + } + } + } } finally { Pop-Location } -if ($Build.IsPresent) { - $macPackages = Get-ChildItem "$repoRoot/powershell*" -Include *.pkg, *.tar.gz +if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { + $macPackages = Get-ChildItem "$repoRoot/powershell*" -Include *.pkg, *.tar.gz, *.zip foreach ($macPackage in $macPackages) { $filePath = $macPackage.FullName - $name = split-path -Leaf -Path $filePath $extension = (Split-Path -Extension -Path $filePath).Replace('.', '') Write-Verbose "Copying $filePath to $destination" -Verbose - Write-Host "##vso[artifact.upload containerfolder=results;artifactname=results]$filePath" + Write-Host "##vso[artifact.upload containerfolder=$ArtifactName;artifactname=$ArtifactName]$filePath" Write-Host "##vso[task.setvariable variable=Package-$extension]$filePath" - Copy-Item -Path $filePath -Destination $destination -force + Copy-Item -Path $filePath -Destination $destination -Force } } diff --git a/tools/releaseBuild/macOS/vsts.yml b/tools/releaseBuild/macOS/vsts.yml deleted file mode 100644 index b00fc9ae0b0..00000000000 --- a/tools/releaseBuild/macOS/vsts.yml +++ /dev/null @@ -1,47 +0,0 @@ -resources: -- repo: self - clean: true -queue: - name: Hosted Mac Internal -name: macOS_$(Build.SourceBranchName)_$(Build.BuildId) -steps: -#- task: @ -# inputs: -# -# displayName: '' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/setReleaseTag.sh' - args: '-ReleaseTag $(ReleaseTagVar) -Variable "ReleaseTagVar"' - displayName: 'Calculate Release Tag' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/installpsh-osx.sh' - displayName: 'Install pwsh' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/macOS/createPowerShell.sh' - displayName: 'Create /PowerShell' -- powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/macOS/PowerShellPackageVsts.sh' - args: '-location $(PowerShellRoot) -BootStrap' - displayName: 'Bootstrap VM' -- powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)/src/Modules" - - if(-not (Test-Path "$(PowerShellRoot)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' -- powershell: | - $env:AZDEVOPSFEEDPAT = '$(AzDevOpsFeedPAT)' - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -ExtraPackage "tar" -location $(PowerShellRoot) -Build - $env:AZDEVOPSFEEDPAT = $null - displayName: 'Build and Package' diff --git a/tools/releaseBuild/setReleaseTag.ps1 b/tools/releaseBuild/setReleaseTag.ps1 index 97ea8ddb86c..5af7db863d7 100644 --- a/tools/releaseBuild/setReleaseTag.ps1 +++ b/tools/releaseBuild/setReleaseTag.ps1 @@ -58,6 +58,8 @@ function New-BuildInfoJson { $branchOnly = $Branch -replace '^refs/heads/'; $branchOnly = $branchOnly -replace '[_\-]' +$msixType = 'preview' + $isDaily = $false if($ReleaseTag -eq 'fromBranch' -or !$ReleaseTag) @@ -65,7 +67,8 @@ if($ReleaseTag -eq 'fromBranch' -or !$ReleaseTag) # Branch is named release- if($Branch -match '^.*(release[-/])') { - Write-verbose "release branch:" -verbose + $msixType = 'release' + Write-Verbose "release branch:" -Verbose $releaseTag = $Branch -replace '^.*(release[-/])' $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose @@ -76,7 +79,7 @@ if($ReleaseTag -eq 'fromBranch' -or !$ReleaseTag) New-BuildInfoJson -ReleaseTag $releaseTag } } - elseif($branchOnly -eq 'master' -or $branchOnly -like '*dailytest*') + elseif(($branchOnly -eq 'master' -and $env:BUILD_REASON -ne 'Manual') -or $branchOnly -like '*dailytest*') { $isDaily = $true Write-verbose "daily build" -verbose @@ -127,4 +130,8 @@ $vstsCommandString = "vso[task.setvariable variable=IS_DAILY]$($isDaily.ToString Write-Verbose -Message "$vstsCommandString" -Verbose Write-Host -Object "##$vstsCommandString" +$vstsCommandString = "vso[task.setvariable variable=MSIX_TYPE]$msixType" +Write-Verbose -Message "$vstsCommandString" -Verbose +Write-Host -Object "##$vstsCommandString" + Write-Output $releaseTag