diff --git a/.spelling b/.spelling index 2f27db0e959..d40538f4ade 100644 --- a/.spelling +++ b/.spelling @@ -21,6 +21,7 @@ AppImage AppVeyor argumentlist arm32 +arm64 artifact artifacts ASP.NET @@ -484,14 +485,18 @@ nanoserver-insider-powershell #region docs/building/internals.md Overrides - docs/building/internals.md +_arm +_arm64 Catalog flavor libpsl MSBuild +nuget.exe plugin powershell-unix psrp.windows src +v141 #endregion #region docs/building/macos.md Overrides diff --git a/docs/building/internals.md b/docs/building/internals.md index 8fdf0fd7334..9880df8dd99 100644 --- a/docs/building/internals.md +++ b/docs/building/internals.md @@ -113,24 +113,41 @@ and then reuse the produced binaries for many builds subsequently. The NuGet package for `pwrshplugin.dll` is `psrp.windows`, and the NuGet package for `libpsl-native` is `libpsl`. -### psrp.windows +### Windows packages: PSRP.Windows and PowerShell.Core.Instrumentation + +To build `pwrshplugin.dll` and `PowerShell.Core.Instrumentation.dll`, you need to install Visual Studio 2017 and run `Start-PSBootstrap -BuildWindowsNative` to install the prerequisites. + +Ensure the following individual components are selected: + +- [ ] VC++ 2017 v141 toolset (x86, x64) +- [ ] Visual C++ compilers and libraries for ARM +- [ ] Visual C++ compilers and libraries for ARM64 +- [ ] Visual C++ tools for CMake +- [ ] Visual C++ ATL Support +- [ ] Windows 10 SDK (10.0.16299.0) for Desktop C++ (ARM and ARM64) +- [ ] Windows 10 SDK (10.0.16299.0) for Desktop C++ (x86 and x64) + +Ensure [CMake](https://cmake.org/download/) 3.10.0 or newer is installed which supports VS2017 and ARM64 generator. -To build `pwrshplugin.dll`, you need to install Visual Studio 2015 and run `Start-PSBootstrap -BuildWindowsNative` to install the prerequisites. Then run `Start-BuildNativeWindowsBinaries` to build the binary. -For example, the following builds the release flavor of the binary targeting x64 architecture. +For example, the following builds the release flavor of the binary targeting arm64 architecture. ```powershell -Start-BuildNativeWindowsBinaries -Configuration Release -Arch x64 +Start-BuildNativeWindowsBinaries -Configuration Release -Arch x64_arm64 ``` -After that, the binary `pwrshplugin.dll` and its PDB file will be placed under 'src/powershell-win-core'. +Be sure to build and test for all supported architectures: x86, x64, x64_arm, and x64_arm64. + +The `x64_arm` and `x64_arm64` architectures mean that the host system needs to be x64 to cross-compile to ARM. +When building for multiple architectures, be sure to use the `-clean` switch as cmake will cache the previous run and the wrong compiler will be used to generate the subsequent architectures. + +After that, the binary `pwrshplugin.dll`, its PDB file, and `powershell.core.instrumentation.dll` will be placed under 'src\powershell-win-core'. To create a new NuGet package for `pwrshplugin.dll`, first you need to get the `psrp.windows.nuspec` from an existing `psrp.windows` package. You can find it at `~/.nuget/packages/psrp.windows` on your windows machine if you have recently built PowerShell on it. Or you can download the existing package from [powershell-core feed](https://powershell.myget.org/feed/powershell-core/package/nuget/psrp.windows). -Once you get `psrp.windows.nuspec`, copy it to an empty folder. +Once you get `psrp.windows.nuspec`, copy it to an empty folder and update the `` element. -Then you need to build `pwrshplugin.dll` targeting both `win-x64` and `win-x86` on Windows 10. After building successfully, copy the produced files to the same folder, and create the same layout of files as in the existing package. The layout of files should look like this: @@ -142,13 +159,29 @@ The layout of files should look like this: | pwrshplugin.dll | pwrshplugin.pdb | - \---win-x86 + +---win-x86 + | \---native + | pwrshplugin.dll + | pwrshplugin.pdb + +---win-arm + | \---native + | pwrshplugin.dll + | pwrshplugin.pdb + \---win-arm64 \---native pwrshplugin.dll pwrshplugin.pdb ``` -Lastly, run `nuget pack .` from within the folder. Note that you may need the latest `nuget.exe`. +Have the DLLs signed with `authenticode dual` certificate and run `nuget pack` from the parent of the `runtimes` folder where `psrp.windows.nuspec` resides. +Be sure to use the latest recommended version of [nuget.exe](https://www.nuget.org/downloads). + +Publish latest nupkg to https://powershell.myget.org/feed/powershell-core/package/nuget/psrp.windows. + +`PowerShell.Core.Instrumentation.dll` NuGet package is created the same way, but in a separate directory following the same layout above. +To create a new NuGet package for `PowerShell.Core.Instrumentation.dll`, you will need the `PowerShell.Core.Instrumentation.nuspec` found in the repo under `src\PowerShell.Core.Instrumentation`. + +Publish latest nupkg to https://powershell.myget.org/feed/powershell-core/package/nuget/PowerShell.Core.Instrumentation. ### libpsl @@ -197,66 +230,3 @@ The layout of files should look like this: ``` Lastly, run `nuget pack .` from within the folder. Note that you may need the latest `nuget.exe`. - -### PowerShell.Core.Instrumentation - -To successfully decode PowerShell Core ETW events, the manifest and resource binary need to be registered on the system. - -To create a new NuGet package for `PowerShell.Core.Instrumentation.dll`, you will need the `PowerShell.Core.Instrumentation.nuspec` found in the repo under `src\PowerShell.Core.Instrumentation`. - -Update the version information for the package. - -```none -6.0.0-RC -``` - -Next, create the directory structure needed for the contents of the nuget package structure. The final directory and file layout is listed below. - -```powershell -if (Test-Path -Path c:\mypackage) -{ - Remove-Item -Recurse -Force -Path c:\mypackage -} -$null = New-Item -Path c:\mypackage\runtimes\win-x64\native -ItemType Directory -$null = New-Item -Path c:\mypackage\runtimes\win-x86\native -ItemType Directory -``` - -You will need to build `PowerShell.Core.Instrumentation.dll` targeting both `win-x64` and `win-x86` on Windows 10. -The output files will be placed under src\powershell-win-core. - -Build the `win-x64` platform and copy the `PowerShell.Core.Instrumentation.dll` to the win-x86 portion of the tree. - -```powershell -## Build targeting win-x64 -Start-BuildNativeWindowsBinaries -Configuration Release -Arch x64 -Copy-Item -Path .\src\powershell-win-core\PowerShell.Core.Instrumentation.dll -Destination c:\mypackage\runtimes\win-x64\native -``` - -Next, build the `win-x86` platform and copy `PowerShell.Core.Instrumentation.dll` to the win-x86 portion of the tree. - -```powershell -## Build targeting win-x86 -Start-BuildNativeWindowsBinaries -Configuration Release -Arch x86 -Copy-Item -Path .\src\powershell-win-core\PowerShell.Core.Instrumentation.dll -Destination c:\mypackage\runtimes\win-x86\native -``` - -The layout of files looks like this: - -```none -└── runtimes - ├── win-x64 - │ └── native - │ └── PowerShell.Core.Instrumentation.dll - │ - ├── win-x86 - │ └── native - │ └── PowerShell.Core.Instrumentation.dll -``` - -NOTE: Since these are native binaries used on Windows, they need to be `authenticode dual signed` before creating the nuget package. - -Lastly, run the following command from the root of the repo to create the nuget package. The nuget package is placed at `.\src\powershell-win-core`. Note that you may need the latest `nuget.exe`. - -```powershell -nuget pack .\src\PowerShell.Core.Instrumentation\PowerShell.Core.Instrumentation.nuspec -BasePath c:\mypackage -OutputDirectory .\src\powershell-win-core -``` diff --git a/tools/installpsh-debian.sh b/tools/installpsh-debian.sh index 972cd187b22..83f27c93e74 100755 --- a/tools/installpsh-debian.sh +++ b/tools/installpsh-debian.sh @@ -14,10 +14,10 @@ #Switches # -includeide - the script is being run headless, do not perform actions that require response from the console # -interactivetests - requires a human user in front of the machine - loads a script into the ide to test with F5 to ensure the IDE can run scripts +# -skip-sudo-check - skips the check that the user has permission to use sudo. This is required to run in the VSTS Hosted Linux Preview. #gitrepo paths are overrideable to run from your own fork or branch for testing or private distribution - VERSION="1.1.2" gitreposubpath="PowerShell/PowerShell/master" gitreposcriptroot="https://raw.githubusercontent.com/$gitreposubpath/tools" @@ -94,7 +94,7 @@ if (( $EUID != 0 )); then fi #Check that sudo is available -if [[ "$SUDO" -eq "sudo" ]]; then +if [[ "$SUDO" -eq "sudo" && ! ("'$*'" =~ skip-sudo-check) ]]; then $SUDO -v if [ $? -ne 0 ]; then diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 080617ae9e3..35f3839e2fe 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -24,7 +24,7 @@ function Start-PSPackage { [string[]]$Type, # Generate windows downlevel package - [ValidateSet("win7-x86", "win7-x64")] + [ValidateSet("win7-x86", "win7-x64", "win-arm", "win-arm64")] [ValidateScript({$Environment.IsWindows})] [string] $WindowsRuntime, @@ -34,8 +34,8 @@ function Start-PSPackage { ) DynamicParam { - if ($Type -eq "zip") { - # Add a dynamic parameter '-IncludeSymbols' when the specified package type is 'zip'. + if ("zip" -eq $Type) { + # Add a dynamic parameter '-IncludeSymbols' when the specified package type is 'zip' only. # The '-IncludeSymbols' parameter can be used to indicate that the package should only contain powershell binaries and symbols. $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute" $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]" @@ -65,9 +65,13 @@ function Start-PSPackage { } if($Environment.IsWindows) { - # Runtime will always be win7-x64 or win7-x86 on Windows. + # Runtime will be one of win7-x64, win7-x86, "win-arm" and "win-arm64" on Windows. # Build the name suffix for universal win-plat packages. - $NameSuffix = $Runtime -replace 'win\d+', 'win' + switch ($Runtime) { + "win-arm" { $NameSuffix = "win-arm32" } + "win-arm64" { $NameSuffix = "win-arm64" } + default { $NameSuffix = $_ -replace 'win\d+', 'win' } + } } log "Packaging RID: '$Runtime'; Packaging Configuration: '$Configuration'" @@ -75,11 +79,11 @@ function Start-PSPackage { $Script:Options = Get-PSOptions $crossGenCorrect = $false - if ($Type -eq "tar-arm") { - # crossgen doesn't support arm32 yet + if ($Runtime -match "arm") { + # crossgen doesn't support arm32/64 $crossGenCorrect = $true } - elseif($Script:Options.CrossGen) { + elseif ($Script:Options.CrossGen) { $crossGenCorrect = $true } @@ -97,6 +101,7 @@ function Start-PSPackage { # Make sure the most recent build satisfies the package requirement if (-not $Script:Options -or ## Start-PSBuild hasn't been executed yet -not $crossGenCorrect -or ## Last build didn't specify '-CrossGen' correctly + -not $PSModuleRestoreCorrect -or ## Last build didn't specify '-PSModuleRestore' correctly $Script:Options.Runtime -ne $Runtime -or ## Last build wasn't for the required RID $Script:Options.Configuration -ne $Configuration -or ## Last build was with configuration other than 'Release' $Script:Options.Framework -ne "netcoreapp2.0") ## Last build wasn't for CoreCLR diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 index 1b3e0688aba..5db03c7eb4a 100644 --- a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 +++ b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 @@ -9,7 +9,7 @@ param ( [string] $destination = "$env:WORKSPACE", - [ValidateSet("win7-x64", "win7-x86")] + [ValidateSet("win7-x64", "win7-x86", "win-arm", "win-arm64")] [string]$Runtime = 'win7-x64', [switch] $Wait, @@ -80,7 +80,7 @@ try{ else { Write-Verbose "Starting powershell build for RID: $Runtime and ReleaseTag: $ReleaseTag ..." -verbose - $buildParams = @{'CrossGen'=$true} + $buildParams = @{'CrossGen'= $Runtime -notmatch "arm"} if(!$Symbols.IsPresent) { $buildParams['PSModuleRestore'] = $true @@ -90,17 +90,14 @@ try{ } $pspackageParams = @{'Type'='msi'; 'WindowsRuntime'=$Runtime} - if(!$Symbols.IsPresent) + if(!$Symbols.IsPresent -and $Runtime -notmatch "arm") { Write-Verbose "Starting powershell packaging(msi)..." -verbose Start-PSPackage @pspackageParams @releaseTagParam } - else - { - $pspackageParams += @{'IncludeSymbols' = $true} - } $pspackageParams['Type']='zip' + $pspackageParams['IncludeSymbols']=$Symbols.IsPresent Write-Verbose "Starting powershell packaging(zip)..." -verbose Start-PSPackage @pspackageParams @releaseTagParam diff --git a/tools/releaseBuild/build.json b/tools/releaseBuild/build.json index 49a6da80cb1..e82c4bafbc9 100644 --- a/tools/releaseBuild/build.json +++ b/tools/releaseBuild/build.json @@ -35,7 +35,7 @@ "BinaryBucket": "release" }, { - "Name": "win7-x64-symbols", + "Name": "win-x64-symbols", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_ -Symbols", "BuildDockerOptions": [ @@ -54,7 +54,7 @@ "VariableForExtractedBinariesPath": "Symbols_x64" }, { - "Name": "win7-x86-symbols", + "Name": "win-x86-symbols", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_ -Symbols", "BuildDockerOptions": [ @@ -73,7 +73,45 @@ "VariableForExtractedBinariesPath": "Symbols_x86" }, { - "Name": "win7-x64-package", + "Name": "win-arm-symbols", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm -ReleaseTag _ReleaseTag_ -Symbols", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "results", + "ArtifactsExpected": 1, + "VariableForExtractedBinariesPath": "Symbols_arm" + }, + { + "Name": "win-arm64-symbols", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm64 -ReleaseTag _ReleaseTag_ -Symbols", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "results", + "ArtifactsExpected": 1, + "VariableForExtractedBinariesPath": "Symbols_arm64" + }, + { + "Name": "win-x64-package", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_", "BuildDockerOptions": [ @@ -91,7 +129,7 @@ "ArtifactsExpected": 2 }, { - "Name": "win7-x86-package", + "Name": "win-x86-package", "RepoDestinationPath": "C:\\PowerShell", "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_", "BuildDockerOptions": [ @@ -107,6 +145,42 @@ "DockerImageName": "ps-winsrvcore", "BinaryBucket": "signed", "ArtifactsExpected": 2 + }, + { + "Name": "win-arm-package", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm -ReleaseTag _ReleaseTag_", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "signed", + "ArtifactsExpected": 1 + }, + { + "Name": "win-arm64-package", + "RepoDestinationPath": "C:\\PowerShell", + "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win-arm64 -ReleaseTag _ReleaseTag_", + "BuildDockerOptions": [ + "-m", + "3968m" + ], + "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile", + "AdditionalContextFiles" :[ + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\wix.psm1", + ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\dockerInstall.psm1" + ], + "DockerImageName": "ps-winsrvcore", + "BinaryBucket": "signed", + "ArtifactsExpected": 1 } ], "Linux": [