From f953156708c09b156a44c6d8c174325813a4a6b5 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 11 Dec 2017 10:28:43 -0800 Subject: [PATCH 1/3] tools: enable install-debian to work with VSTS Hosted Linux Preview (#5659) Added -skip-sudo-check, because the check fails in VSTS Hosted Linux Preview even though sudo works. --- tools/installpsh-debian.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/installpsh-debian.sh b/tools/installpsh-debian.sh index 972cd187b22..83f27c93e74 100755 --- a/tools/installpsh-debian.sh +++ b/tools/installpsh-debian.sh @@ -14,10 +14,10 @@ #Switches # -includeide - the script is being run headless, do not perform actions that require response from the console # -interactivetests - requires a human user in front of the machine - loads a script into the ide to test with F5 to ensure the IDE can run scripts +# -skip-sudo-check - skips the check that the user has permission to use sudo. This is required to run in the VSTS Hosted Linux Preview. #gitrepo paths are overrideable to run from your own fork or branch for testing or private distribution - VERSION="1.1.2" gitreposubpath="PowerShell/PowerShell/master" gitreposcriptroot="https://raw.githubusercontent.com/$gitreposubpath/tools" @@ -94,7 +94,7 @@ if (( $EUID != 0 )); then fi #Check that sudo is available -if [[ "$SUDO" -eq "sudo" ]]; then +if [[ "$SUDO" -eq "sudo" && ! ("'$*'" =~ skip-sudo-check) ]]; then $SUDO -v if [ $? -ne 0 ]; then From a52b010e26fcb994389e4e251e39f4945da015cc Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Mon, 11 Dec 2017 13:48:24 -0800 Subject: [PATCH 2/3] Updated doc to build arm/arm64 versions of psrp.windows and PowerShell.Core.Instrumentation.dll libraries (#5668) --- .spelling | 5 ++ docs/building/internals.md | 114 ++++++++++++++----------------------- 2 files changed, 47 insertions(+), 72 deletions(-) diff --git a/.spelling b/.spelling index 2f27db0e959..d40538f4ade 100644 --- a/.spelling +++ b/.spelling @@ -21,6 +21,7 @@ AppImage AppVeyor argumentlist arm32 +arm64 artifact artifacts ASP.NET @@ -484,14 +485,18 @@ nanoserver-insider-powershell #region docs/building/internals.md Overrides - docs/building/internals.md +_arm +_arm64 Catalog flavor libpsl MSBuild +nuget.exe plugin powershell-unix psrp.windows src +v141 #endregion #region docs/building/macos.md Overrides diff --git a/docs/building/internals.md b/docs/building/internals.md index 8fdf0fd7334..9880df8dd99 100644 --- a/docs/building/internals.md +++ b/docs/building/internals.md @@ -113,24 +113,41 @@ and then reuse the produced binaries for many builds subsequently. The NuGet package for `pwrshplugin.dll` is `psrp.windows`, and the NuGet package for `libpsl-native` is `libpsl`. -### psrp.windows +### Windows packages: PSRP.Windows and PowerShell.Core.Instrumentation + +To build `pwrshplugin.dll` and `PowerShell.Core.Instrumentation.dll`, you need to install Visual Studio 2017 and run `Start-PSBootstrap -BuildWindowsNative` to install the prerequisites. + +Ensure the following individual components are selected: + +- [ ] VC++ 2017 v141 toolset (x86, x64) +- [ ] Visual C++ compilers and libraries for ARM +- [ ] Visual C++ compilers and libraries for ARM64 +- [ ] Visual C++ tools for CMake +- [ ] Visual C++ ATL Support +- [ ] Windows 10 SDK (10.0.16299.0) for Desktop C++ (ARM and ARM64) +- [ ] Windows 10 SDK (10.0.16299.0) for Desktop C++ (x86 and x64) + +Ensure [CMake](https://cmake.org/download/) 3.10.0 or newer is installed which supports VS2017 and ARM64 generator. -To build `pwrshplugin.dll`, you need to install Visual Studio 2015 and run `Start-PSBootstrap -BuildWindowsNative` to install the prerequisites. Then run `Start-BuildNativeWindowsBinaries` to build the binary. -For example, the following builds the release flavor of the binary targeting x64 architecture. +For example, the following builds the release flavor of the binary targeting arm64 architecture. ```powershell -Start-BuildNativeWindowsBinaries -Configuration Release -Arch x64 +Start-BuildNativeWindowsBinaries -Configuration Release -Arch x64_arm64 ``` -After that, the binary `pwrshplugin.dll` and its PDB file will be placed under 'src/powershell-win-core'. +Be sure to build and test for all supported architectures: x86, x64, x64_arm, and x64_arm64. + +The `x64_arm` and `x64_arm64` architectures mean that the host system needs to be x64 to cross-compile to ARM. +When building for multiple architectures, be sure to use the `-clean` switch as cmake will cache the previous run and the wrong compiler will be used to generate the subsequent architectures. + +After that, the binary `pwrshplugin.dll`, its PDB file, and `powershell.core.instrumentation.dll` will be placed under 'src\powershell-win-core'. To create a new NuGet package for `pwrshplugin.dll`, first you need to get the `psrp.windows.nuspec` from an existing `psrp.windows` package. You can find it at `~/.nuget/packages/psrp.windows` on your windows machine if you have recently built PowerShell on it. Or you can download the existing package from [powershell-core feed](https://powershell.myget.org/feed/powershell-core/package/nuget/psrp.windows). -Once you get `psrp.windows.nuspec`, copy it to an empty folder. +Once you get `psrp.windows.nuspec`, copy it to an empty folder and update the `` element. -Then you need to build `pwrshplugin.dll` targeting both `win-x64` and `win-x86` on Windows 10. After building successfully, copy the produced files to the same folder, and create the same layout of files as in the existing package. The layout of files should look like this: @@ -142,13 +159,29 @@ The layout of files should look like this: | pwrshplugin.dll | pwrshplugin.pdb | - \---win-x86 + +---win-x86 + | \---native + | pwrshplugin.dll + | pwrshplugin.pdb + +---win-arm + | \---native + | pwrshplugin.dll + | pwrshplugin.pdb + \---win-arm64 \---native pwrshplugin.dll pwrshplugin.pdb ``` -Lastly, run `nuget pack .` from within the folder. Note that you may need the latest `nuget.exe`. +Have the DLLs signed with `authenticode dual` certificate and run `nuget pack` from the parent of the `runtimes` folder where `psrp.windows.nuspec` resides. +Be sure to use the latest recommended version of [nuget.exe](https://www.nuget.org/downloads). + +Publish latest nupkg to https://powershell.myget.org/feed/powershell-core/package/nuget/psrp.windows. + +`PowerShell.Core.Instrumentation.dll` NuGet package is created the same way, but in a separate directory following the same layout above. +To create a new NuGet package for `PowerShell.Core.Instrumentation.dll`, you will need the `PowerShell.Core.Instrumentation.nuspec` found in the repo under `src\PowerShell.Core.Instrumentation`. + +Publish latest nupkg to https://powershell.myget.org/feed/powershell-core/package/nuget/PowerShell.Core.Instrumentation. ### libpsl @@ -197,66 +230,3 @@ The layout of files should look like this: ``` Lastly, run `nuget pack .` from within the folder. Note that you may need the latest `nuget.exe`. - -### PowerShell.Core.Instrumentation - -To successfully decode PowerShell Core ETW events, the manifest and resource binary need to be registered on the system. - -To create a new NuGet package for `PowerShell.Core.Instrumentation.dll`, you will need the `PowerShell.Core.Instrumentation.nuspec` found in the repo under `src\PowerShell.Core.Instrumentation`. - -Update the version information for the package. - -```none -6.0.0-RC -``` - -Next, create the directory structure needed for the contents of the nuget package structure. The final directory and file layout is listed below. - -```powershell -if (Test-Path -Path c:\mypackage) -{ - Remove-Item -Recurse -Force -Path c:\mypackage -} -$null = New-Item -Path c:\mypackage\runtimes\win-x64\native -ItemType Directory -$null = New-Item -Path c:\mypackage\runtimes\win-x86\native -ItemType Directory -``` - -You will need to build `PowerShell.Core.Instrumentation.dll` targeting both `win-x64` and `win-x86` on Windows 10. -The output files will be placed under src\powershell-win-core. - -Build the `win-x64` platform and copy the `PowerShell.Core.Instrumentation.dll` to the win-x86 portion of the tree. - -```powershell -## Build targeting win-x64 -Start-BuildNativeWindowsBinaries -Configuration Release -Arch x64 -Copy-Item -Path .\src\powershell-win-core\PowerShell.Core.Instrumentation.dll -Destination c:\mypackage\runtimes\win-x64\native -``` - -Next, build the `win-x86` platform and copy `PowerShell.Core.Instrumentation.dll` to the win-x86 portion of the tree. - -```powershell -## Build targeting win-x86 -Start-BuildNativeWindowsBinaries -Configuration Release -Arch x86 -Copy-Item -Path .\src\powershell-win-core\PowerShell.Core.Instrumentation.dll -Destination c:\mypackage\runtimes\win-x86\native -``` - -The layout of files looks like this: - -```none -└── runtimes - ├── win-x64 - │ └── native - │ └── PowerShell.Core.Instrumentation.dll - │ - ├── win-x86 - │ └── native - │ └── PowerShell.Core.Instrumentation.dll -``` - -NOTE: Since these are native binaries used on Windows, they need to be `authenticode dual signed` before creating the nuget package. - -Lastly, run the following command from the root of the repo to create the nuget package. The nuget package is placed at `.\src\powershell-win-core`. Note that you may need the latest `nuget.exe`. - -```powershell -nuget pack .\src\PowerShell.Core.Instrumentation\PowerShell.Core.Instrumentation.nuspec -BasePath c:\mypackage -OutputDirectory .\src\powershell-win-core -``` From c00909e5bfa49e99d8a871ae80fb26d021329fe5 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 11 Dec 2017 13:57:02 -0800 Subject: [PATCH 3/3] Update packaging and release build scripts to produce zip packages for win-arm/win-arm64 (#5664) Note: changes in tools/appveyor.psm1 are not moved to the 6.0.0 branch because: - the change is to produce win-arm32/win-arm64 zip packages in daily builds. Since we don't have daily build for the 6.0.0 branch, the change is not needed. - there are conflicts when cherry-picking the commits --- tools/packaging/packaging.psm1 | 21 +++-- .../PowerShellPackage.ps1 | 11 +-- tools/releaseBuild/build.json | 82 ++++++++++++++++++- 3 files changed, 95 insertions(+), 19 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 080617ae9e3..35f3839e2fe 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -24,7 +24,7 @@ function Start-PSPackage { [string[]]$Type, # Generate windows downlevel package - [ValidateSet("win7-x86", "win7-x64")] + [ValidateSet("win7-x86", "win7-x64", "win-arm", "win-arm64")] [ValidateScript({$Environment.IsWindows})] [string] $WindowsRuntime, @@ -34,8 +34,8 @@ function Start-PSPackage { ) DynamicParam { - if ($Type -eq "zip") { - # Add a dynamic parameter '-IncludeSymbols' when the specified package type is 'zip'. + if ("zip" -eq $Type) { + # Add a dynamic parameter '-IncludeSymbols' when the specified package type is 'zip' only. # The '-IncludeSymbols' parameter can be used to indicate that the package should only contain powershell binaries and symbols. $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute" $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]" @@ -65,9 +65,13 @@ function Start-PSPackage { } if($Environment.IsWindows) { - # Runtime will always be win7-x64 or win7-x86 on Windows. + # Runtime will be one of win7-x64, win7-x86, "win-arm" and "win-arm64" on Windows. # Build the name suffix for universal win-plat packages. - $NameSuffix = $Runtime -replace 'win\d+', 'win' + switch ($Runtime) { + "win-arm" { $NameSuffix = "win-arm32" } + "win-arm64" { $NameSuffix = "win-arm64" } + default { $NameSuffix = $_ -replace 'win\d+', 'win' } + } } log "Packaging RID: '$Runtime'; Packaging Configuration: '$Configuration'" @@ -75,11 +79,11 @@ function Start-PSPackage { $Script:Options = Get-PSOptions $crossGenCorrect = $false - if ($Type -eq "tar-arm") { - # crossgen doesn't support arm32 yet + if ($Runtime -match "arm") { + # crossgen doesn't support arm32/64 $crossGenCorrect = $true } - elseif($Script:Options.CrossGen) { + elseif ($Script:Options.CrossGen) { $crossGenCorrect = $true } @@ -97,6 +101,7 @@ function Start-PSPackage { # Make sure the most recent build satisfies the package requirement if (-not $Script:Options -or ## Start-PSBuild hasn't been executed yet -not $crossGenCorrect -or ## Last build didn't specify '-CrossGen' correctly + -not $PSModuleRestoreCorrect -or ## Last build didn't specify '-PSModuleRestore' correctly $Script:Options.Runtime -ne $Runtime -or ## Last build wasn't for the required RID $Script:Options.Configuration -ne $Configuration -or ## Last build was with configuration other than 'Release' $Script:Options.Framework -ne "netcoreapp2.0") ## Last build wasn't for CoreCLR diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 index 1b3e0688aba..5db03c7eb4a 100644 --- a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 +++ b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 @@ -9,7 +9,7 @@ param ( [string] $destination = "$env:WORKSPACE", - [ValidateSet("win7-x64", "win7-x86")] + [ValidateSet("win7-x64", "win7-x86", "win-arm", "win-arm64")] [string]$Runtime = 'win7-x64', [switch] $Wait, @@ -80,7 +80,7 @@ try{ else { Write-Verbose "Starting powershell build for RID: $Runtime and ReleaseTag: $ReleaseTag ..." -verbose - $buildParams = @{'CrossGen'=$true} + $buildParams = @{'CrossGen'= $Runtime -notmatch "arm"} if(!$Symbols.IsPresent) { $buildParams['PSModuleRestore'] = $true @@ -90,17 +90,14 @@ try{ } $pspackageParams = @{'Type'='msi'; 'WindowsRuntime'=$Runtime} - if(!$Symbols.IsPresent) + if(!$Symbols.IsPresent -and $Runtime -notmatch "arm") { Write-Verbose "Starting powershell packaging(msi)..." -verbose Start-PSPackage @pspackageParams @releaseTagParam } - else - { - $pspackageParams += @{'IncludeSymbols' = $true} - } $pspackageParams['Type']='zip' + $pspackageParams['IncludeSymbols']=$Symbols.IsPresent Write-Verbose "Starting powershell packaging(zip)..." -verbose Start-PSPackage @pspackageParams @releaseTagParam diff --git a/tools/releaseBuild/build.json b/tools/releaseBuild/build.json index 49a6da80cb1..e82c4bafbc9 100644 --- a/tools/releaseBuild/build.json +++ b/tools/releaseBuild/build.json @@ -35,7 +35,7 @@ "BinaryBucket": "release" }, { - "Name": "win7-x64-symbols", + "Name": "win-x64-symbols", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_ -Symbols", "BuildDockerOptions": [ @@ -54,7 +54,7 @@ "VariableForExtractedBinariesPath": "Symbols_x64" }, { - "Name": "win7-x86-symbols", + "Name": "win-x86-symbols", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_ -Symbols", "BuildDockerOptions": [ @@ -73,7 +73,45 @@ "VariableForExtractedBinariesPath": "Symbols_x86" }, { - "Name": "win7-x64-package", + "Name": "win-arm-symbols", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm -ReleaseTag _ReleaseTag_ -Symbols", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "results", + "ArtifactsExpected": 1, + "VariableForExtractedBinariesPath": "Symbols_arm" + }, + { + "Name": "win-arm64-symbols", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm64 -ReleaseTag _ReleaseTag_ -Symbols", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "results", + "ArtifactsExpected": 1, + "VariableForExtractedBinariesPath": "Symbols_arm64" + }, + { + "Name": "win-x64-package", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_", "BuildDockerOptions": [ @@ -91,7 +129,7 @@ "ArtifactsExpected": 2 }, { - "Name": "win7-x86-package", + "Name": "win-x86-package", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_", "BuildDockerOptions": [ @@ -107,6 +145,42 @@ "DockerImageName": "ps-winsrvcore", "BinaryBucket": "signed", "ArtifactsExpected": 2 + }, + { + "Name": "win-arm-package", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm -ReleaseTag _ReleaseTag_", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "signed", + "ArtifactsExpected": 1 + }, + { + "Name": "win-arm64-package", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm64 -ReleaseTag _ReleaseTag_", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "signed", + "ArtifactsExpected": 1 } ], "Linux": [