From 242b50f64a3c7098c8f159e6e4c3553ed8a37968 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 3 May 2024 13:16:02 -0700 Subject: [PATCH] Create the Windows.x64 global tool with shim for signing (#21559) --- .pipelines/templates/nupkg.yml | 9 +- .pipelines/templates/obp-file-signing.yml | 16 +- .pipelines/templates/windows-hosted-build.yml | 143 ++++++++++++++++++ .../PowerShell.Windows.x64.csproj | 31 ++++ .../PowerShell.Windows.x64/Powershell_64.png | Bin 0 -> 2201 bytes tools/packaging/packaging.psm1 | 30 ++-- tools/packaging/packaging.strings.psd1 | 23 +++ 7 files changed, 233 insertions(+), 19 deletions(-) create mode 100644 src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj create mode 100644 src/GlobalTools/PowerShell.Windows.x64/Powershell_64.png diff --git a/.pipelines/templates/nupkg.yml b/.pipelines/templates/nupkg.yml index 427654e2516..9b666d4e578 100644 --- a/.pipelines/templates/nupkg.yml +++ b/.pipelines/templates/nupkg.yml @@ -258,7 +258,8 @@ jobs: $linuxFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd\Signed-fxdependent" $alpineFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd_x64_alpine\Signed-fxdependent-noopt-linux-musl-x64" - $packageTypes = @('Unified', 'PowerShell.Linux.Alpine', 'PowerShell.Linux.x64', 'PowerShell.Linux.arm32', 'PowerShell.Linux.arm64', 'PowerShell.Windows.x64') + # Build global tools which do not have the shims exe generated in build. + $packageTypes = @('Unified', 'PowerShell.Linux.Alpine', 'PowerShell.Linux.x64', 'PowerShell.Linux.arm32', 'PowerShell.Linux.arm64') $packageTypes | Foreach-Object { $PackageType = $_ @@ -299,6 +300,12 @@ jobs: $nupkgOutputPath = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'nupkg' Get-ChildItem -Path $nupkgOutputPath -Filter *.nupkg -Recurse | Copy-Item -Destination '$(ob_outputDirectory)' -Force -Verbose + # Copy Windows.x86 global tool from build to output directory + $winX64GlobalTool = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependent_release\globaltool\powershell*.nupkg" + Write-Verbose -Verbose "Finding Windows.x64 global tool at $winX64GlobalTool" + $globalToolPath = Get-Item $winX64GlobalTool + Copy-Item -Path $globalToolPath -Destination '$(ob_outputDirectory)' -Force -Verbose + Write-Verbose -Verbose "Copying global tools to output directory" $gblToolOutputPath = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'globaltools' Get-ChildItem -Path $gblToolOutputPath -Filter *.nupkg -Recurse | Copy-Item -Destination '$(ob_outputDirectory)' -Force -Verbose diff --git a/.pipelines/templates/obp-file-signing.yml b/.pipelines/templates/obp-file-signing.yml index 06cd46dec29..ab750c2700f 100644 --- a/.pipelines/templates/obp-file-signing.yml +++ b/.pipelines/templates/obp-file-signing.yml @@ -1,5 +1,6 @@ parameters: binPath: '$(ob_outputDirectory)' + globalTool: 'false' steps: - pwsh: | @@ -138,10 +139,17 @@ steps: - pwsh: | Import-Module '$(PowerShellRoot)/build.psm1' -Force Import-Module '$(PowerShellRoot)/tools/packaging' -Force - $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)/Signed-$(Runtime)' -Force - Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" - Copy-Item -Path '${{ parameters.binPath }}\*' -Destination $pathForUpload -Recurse -Force -Verbose - Write-Verbose -Verbose -Message "Files copied to $pathForUpload" + $isGlobalTool = '${{ parameters.globalTool }}' -eq 'true' + + if (-not $isGlobalTool) { + $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)/Signed-$(Runtime)' -Force + Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" + Copy-Item -Path '${{ parameters.binPath }}\*' -Destination $pathForUpload -Recurse -Force -Verbose + Write-Verbose -Verbose -Message "Files copied to $pathForUpload" + } + else { + $pathForUpload = '${{ parameters.binPath }}' + } Write-Verbose "Copying third party signed files to the build folder" $thirdPartySignedFilesPath = (Get-Item '$(Pipeline.Workspace)/thirdPartyToBeSigned').FullName diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 15ca7e9c587..d6be94368b6 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -89,6 +89,8 @@ jobs: $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params + $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' + Write-Verbose -Verbose "refFolderPath: $refFolderPath" $outputPath = Join-Path '$(ob_outputDirectory)' 'psoptions' $null = New-Item -ItemType Directory -Path $outputPath -Force $psOptPath = "$outputPath/psoptions.json" @@ -106,6 +108,39 @@ jobs: } } + if ($runtime -eq 'fxdependent') + { + ## Also build global tool + Write-Verbose -Message "Building PowerShell global tool for Windows.x64" -Verbose + $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' + Push-Location -Path $globalToolCsProjDir -Verbose + + $globalToolArtifactPath = Join-Path $(Build.SourcesDirectory) 'GlobalTool' + $vstsCommandString = "vso[task.setvariable variable=GlobalToolArtifactPath]${globalToolArtifactPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) + $globalToolBuildModulePath = Join-Path $globalToolArtifactPath 'publish' 'PowerShell.Windows.x64' 'release' + Pop-Location + # do this to ensure everything gets signed. + Restore-PSModuleToBuild -PublishPath $globalToolBuildModulePath + + # Copy reference assemblies + Copy-Item -Path $refFolderPath -Destination $globalToolBuildModulePath -Recurse -Force + + Write-Verbose -Verbose "clean unnecessary files in obj directory" + $objDir = Join-Path $globalToolArtifactPath 'obj' 'PowerShell.Windows.x64' 'release' + + $filesToKeep = @("apphost.exe", "PowerShell.Windows.x64.pdb", "PowerShell.Windows.x64.dll", "project.assets.json") + + # only four files are needed in obj folder for global tool packaging + Get-ChildItem -Path $objDir -File -Recurse | + Where-Object { -not $_.PSIsContainer } | + Where-Object { $_.name -notin $filesToKeep } | + Remove-Item -Verbose + } + Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" displayName: 'Build Windows Universal - $(Architecture)-$(BuildConfiguration) Symbols folder' env: @@ -128,4 +163,112 @@ jobs: parameters: binPath: '$(Pipeline.Workspace)/Symbols_$(Architecture)' + ## first we sign all the files in the bin folder + - ${{ if eq(variables['Architecture'], 'fxdependent') }}: + - template: /.pipelines/templates/obp-file-signing.yml@self + parameters: + binPath: '$(GlobalToolArtifactPath)/publish/PowerShell.Windows.x64/release' + globalTool: 'true' + + - pwsh: | + Get-ChildItem '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' + displayName: Capture obj files + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + + ## Now we sign couple of file from the obj folder which are needed for the global tool packaging + - task: onebranch.pipeline.signing@1 + displayName: Sign obj files + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '**\*.dll;**\*.exe' + search_root: '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + + - pwsh: | + <# The way the packaging works is a bit tricky as when it is built, we cannot add the modules that come from gallery. + We have to use dotnet pack to build the nupkg and then expand it as a zip. + After expanding we restore the signed files for the modules from the gallery. + We also delete pdbs, content and contentFiles folder which are not necessary. + After that, we repack using Compress-Archive and rename it back to a nupkg. + #> + + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + Start-PSBootstrap + $packagingStrings = Import-PowerShellDataFile "$(PowerShellRoot)\tools\packaging\packaging.strings.psd1" + + $outputPath = Join-Path '$(ob_outputDirectory)' 'globaltool' + $null = New-Item -ItemType Directory -Path $outputPath -Force + $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' + Push-Location -Path $globalToolCsProjDir -Verbose + + <# + $nuspecFilePath = "$globalToolCsProjDir\PowerShell.Windows.x64.nuspec" + $nuSpec = $packagingStrings.WindowsX64GlobalToolNuspec -f '$(Version)' + $nuSpec | Out-File -FilePath $nuspecFilePath -Encoding ascii + #> + + dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png + + Write-Verbose -Verbose "Deleting content and contentFiles folders from the nupkg" + + $nupkgs = Get-ChildItem -Path $outputPath -Filter powershell*.nupkg + + $nupkgName = $nupkgs.Name + $newName = $nupkgName -replace '(\.nupkg)$', '.zip' + Rename-Item -Path $nupkgs.FullName -NewName $newName + + $zipPath = Get-ChildItem -Path $outputPath -Filter powershell*.zip + + # Expand zip and remove content and contentFiles folders + Expand-Archive -Path $zipPath -DestinationPath "$outputPath\temp" -Force + + $modulesToCopy = @( + 'PowerShellGet' + 'PackageManagement' + 'Microsoft.PowerShell.PSResourceGet' + 'Microsoft.PowerShell.Archive' + 'PSReadLine' + 'ThreadJob' + ) + + $sourceModulePath = Join-Path '$(GlobalToolArtifactPath)' 'publish' 'PowerShell.Windows.x64' 'release' 'Modules' + $destModulesPath = Join-Path "$outputPath" 'temp' 'tools' 'net9.0' 'any' 'modules' + + $modulesToCopy | ForEach-Object { + $modulePath = Join-Path $sourceModulePath $_ + Copy-Item -Path $modulePath -Destination $destModulesPath -Recurse -Force + } + + # Copy ref assemblies + Copy-Item '$(Pipeline.Workspace)/Symbols_$(Architecture)/ref' "$outputPath\temp\tools\net9.0\any\ref" -Recurse -Force + + $contentPath = Join-Path "$outputPath\temp" 'content' + $contentFilesPath = Join-Path "$outputPath\temp" 'contentFiles' + + Remove-Item -Path $contentPath,$contentFilesPath -Recurse -Force + + # remove PDBs to reduce the size of the nupkg + Remove-Item -Path "$outputPath\temp\tools\net9.0\any\*.pdb" -Recurse -Force + + Compress-Archive -Path "$outputPath\temp\*" -DestinationPath "$outputPath\$nupkgName" -Force + + Remove-Item -Path "$outputPath\temp" -Recurse -Force + Remove-Item -Path $zipPath -Force + + if (-not (Test-Path "$outputPath\powershell.windows.x64.*.nupkg")) { + throw "Global tool package not found at $outputPath" + } + displayName: 'Pack Windows.x64 global tool' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + + - task: onebranch.pipeline.signing@1 + displayName: Sign nupkg files + inputs: + command: 'sign' + cp_code: 'CP-401405' + files_to_sign: '**\*.nupkg' + search_root: '$(ob_outputDirectory)\globaltool' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj b/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj new file mode 100644 index 00000000000..045faad6144 --- /dev/null +++ b/src/GlobalTools/PowerShell.Windows.x64/PowerShell.Windows.x64.csproj @@ -0,0 +1,31 @@ + + + + Exe + net9.0 + enable + enable + true + win-x64 + pwsh + $(PackageVersion) + + + + + + Modules\%(RecursiveDir)\%(FileName)%(Extension) + PreserveNewest + PreserveNewest + + + + + + + + + + + + diff --git a/src/GlobalTools/PowerShell.Windows.x64/Powershell_64.png b/src/GlobalTools/PowerShell.Windows.x64/Powershell_64.png new file mode 100644 index 0000000000000000000000000000000000000000..2a656ffc3c83f404f55f8aa3ee842c53a2d5a9e5 GIT binary patch literal 2201 zcmV;K2xj+*P)1A05Sk%0LTE40bl|T34ahmm@c08`YD+ncG2%Osuxf>_=U%WInIWUhSv}w?wnbf zg3f)4%mDp;Wz*(KA&C>P!;;PI_f9SYQL-ofDFOl;8!Wy3u0YQ+!>YC&Sf#4$UW_AI(`YVVa`Q<6)D#}Nnhgo@0QFQn$#O6F;CdrFE zi<-Rf&;UfjgJ=daZw#okdkd+Bsu)e|!(ty4|9}u-UfI~gjvQH)SCbbnXN){D;`TfM z_#~T4Tlpjfv`t-aWs~m((_re!6GU7@5*P`BjKgnn$hykM=JU%$nE$v4?f1iQ@Yexo z4~3IfhbKmWKA4>H(1M9zK_dt<13>-XFIw!0zOUhfFZ(4}Hb)%EvK6Hwtot$qce@4c z2juo?1BG?uDLN1mY}zD;U}J>Y1^klLt*v~V8ICF+9WEPzP1P2#3lBLmf}T&C$uOgH zL9(Kx051_Lw>Yv4mDg&nOrPS0?Jwnl(+1|0=Lxa~@Gx^gT4G66vS-$xHJS^!IK{^VF6{M{Zl1i?IU z0#4V7SKfh-(RyT~x0S;Jh);?Bj6|omFAQ4__EHcuETKSxHP0X+8B7b&(UT`yok{@Y zD~1SES}6cLM5l*>@XpsgnjkEFIuGh=a??n3)a6N57gC^RE&%8aeg!#BqJx(3!J%&G z?N<+s3uilE{c<@iL`O@W;{RmS{|y1Ce4|;WHxyg!-Z3V}LQLAbhT!e}T@(y;5UR?Z zP*mw;6I$Fd@w7L?a zKcfo}AKo}#VsgE05H|0-r?`dMc&&OWESPOi3DJpYq(jUAH2*hq0YGo4wAeid zJ?R-2(NPPEgou;^+xT3-zy6G$(6UhwEevJ|%&YTbum?Um@~(e}^0FA$AKOSpR!WcXafKlLA6xhvrA`tSzOiQA$~dNJ2LT66EX(lgcP{SnA>}vs=cjmY|L=yHfi2CYl13>e?U`Z`8 z$(JSA`a-_0+2c5R`Woy!a1OvRjbC^adZAe=0gKctpCpc4>~T$#*{( zfcN&ChBKEs!JR!_Nx#M&BICuuN;bI!)&*ce!&xkVCerw z5cW1*22XA&IJ^#1L`RF9Mca4~Fnl>WrT|in{_I3&wLc41S7zxde!HUww(a>vp~rGT znetLJl^r4+!;Rc#J^%<*D!kbQ(ZSq~t3IpPuB&|D{7u-l_cw@G3&A^SDwj_)b%A=l|UyKJdvJPVR6G6eT^5S0WidF0ki-@^N%q0-%AQ4 zsH@IV0-j-@J9-5?IVIrC@)%|lQ?J636|)(ezlsZxsQ)wM`kD$Cte)pJH2H>)Pr}94 ze#kDEMJo?sMBbvOWO#MiWN_O>?rkmG5Bvo`w6u*0dhvrL0FEh!2QI$uR{7YFZQan(E$b;CqW;h00#O3Q-fvS6beP+3cG@7jaF%Wa z?|C=`Zijgg{<%YAo2U)|OY|S>0)&*0=6`*ngALB)ap3r6She+gxYJKfJGZWa14&#?Q3V!2hMOOicGZNQu01~tV>kc?SluZW1N>uNf0=w0mbN`VtXp4|S765JFA)Gjij*y< z|HB>~Q}P%G@_*dR$U=Y;_X-2Kplme_4@LLWrg4uBw>-vyzV9t8Ll6Sg{~Nggph|y5 zf7T2QKJaU8XvX+}bAfr&Gv_Ah*fna`vk%mxj^_rF`M0?VJ~9Ag0LTE40U!fF27n1X b{u5vTy2wFLV%}Mq00000NkvXXu0mjfvw`l| literal 0 HcmV?d00001 diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index f7295bffcf1..8ba16f9e284 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -4314,20 +4314,22 @@ function New-GlobalToolNupkgSource $toolSettings = $packagingStrings.GlobalToolSettingsFile -f "pwsh.dll" } - "PowerShell.Windows.x64" - { - $PackageName = "PowerShell.Windows.x64" - $RootFolder = New-TempFolder - - Copy-Item -Path $iconPath -Destination "$RootFolder/$iconFileName" -Verbose - - $ridFolder = New-Item -Path (Join-Path $RootFolder "tools/$script:netCoreRuntime/any") -ItemType Directory - - Write-Log "New-GlobalToolNupkgSource: Copying runtime assemblies from $WindowsDesktopBinPath for $PackageType" - Copy-Item "$WindowsDesktopBinPath/*" -Destination $ridFolder -Recurse - Remove-Item -Path $ridFolder/runtimes/win-arm -Recurse -Force - $toolSettings = $packagingStrings.GlobalToolSettingsFile -f "pwsh.dll" - } + # Due to needing a signed shim for the global tool, we build the global tool in build instead of packaging. + # keeping the code for reference. + # "PowerShell.Windows.x64" + # { + # $PackageName = "PowerShell.Windows.x64" + # $RootFolder = New-TempFolder + + # Copy-Item -Path $iconPath -Destination "$RootFolder/$iconFileName" -Verbose + + # $ridFolder = New-Item -Path (Join-Path $RootFolder "tools/$script:netCoreRuntime/any") -ItemType Directory + + # Write-Log "New-GlobalToolNupkgSource: Copying runtime assemblies from $WindowsDesktopBinPath for $PackageType" + # Copy-Item "$WindowsDesktopBinPath/*" -Destination $ridFolder -Recurse + # Remove-Item -Path $ridFolder/runtimes/win-arm -Recurse -Force + # $toolSettings = $packagingStrings.GlobalToolSettingsFile -f "pwsh.dll" + # } "PowerShell.Windows.arm32" { diff --git a/tools/packaging/packaging.strings.psd1 b/tools/packaging/packaging.strings.psd1 index 39afb75a96b..adf39654d95 100644 --- a/tools/packaging/packaging.strings.psd1 +++ b/tools/packaging/packaging.strings.psd1 @@ -193,6 +193,29 @@ open {0} +'@ + + WindowsX64GlobalToolNuspec = @' + + + + PowerShelll.Windows.x64 + {0} + Microsoft + Microsoft,PowerShell + https://github.com/PowerShell/PowerShell + Powershell_64.png + false + PowerShell global tool + MIT + PowerShell + en-US + © Microsoft Corporation. All rights reserved. + + + + + '@ GlobalToolSettingsFile = @'