diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index adc614f3284..77ac93401d2 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -68,6 +68,7 @@ variables: - name: SKIP_SIGNING value: ${{ parameters.SKIP_SIGNING }} - group: 'AzDevOpsArtifacts' + - group: 'mscodehub-feed-read-akv' extends: template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates diff --git a/.pipelines/PowerShell-Packages-Official.yml b/.pipelines/PowerShell-Packages-Official.yml new file mode 100644 index 00000000000..edd4692298a --- /dev/null +++ b/.pipelines/PowerShell-Packages-Official.yml @@ -0,0 +1,223 @@ +trigger: none # https://aka.ms/obpipelines/triggers + +parameters: # parameters are shown up in ADO UI in a build queue time + - name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning + - name: system.debug + value: ${{ parameters.debug }} + - name: ENABLE_PRS_DELAYSIGN + value: 1 + - name: ROOT + value: $(Build.SourcesDirectory) + - name: NUGET_XMLDOC_MODE + value: none + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project https://aka.ms/obpipelines/containers + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + - group: mscodehub-feed-read-akv + +resources: + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated Packages-Official' + trigger: + branches: + include: + - master + - releases/* + + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/OneBranch.Official.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates + parameters: + cloudvault: # https://aka.ms/obpipelines/cloudvault + enabled: false + featureFlags: + linuxEsrpSigning: true + globalSdl: + disableLegacyManifest: true + # disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + sbom: + enabled: true + compiled: + enabled: false + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + cg: + enabled: true + ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' + asyncSdl: # https://aka.ms/obpipelines/asyncsdl + enabled: true + forStages: ['build'] + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + binskim: + enabled: false + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: false + tsaOptionsFile: .config\tsaoptions.json + stages: + - stage: mac_package + jobs: + - template: /.pipelines/templates/mac-package-build.yml@self + parameters: + buildArchitecture: x64 + + - template: /.pipelines/templates/mac-package-build.yml@self + parameters: + buildArchitecture: arm64 + + - stage: windows_package + jobs: + - template: /.pipelines/templates/windows-package-build.yml@self + parameters: + runtime: x64 + + - template: /.pipelines/templates/windows-package-build.yml@self + parameters: + runtime: arm64 + + - template: /.pipelines/templates/windows-package-build.yml@self + parameters: + runtime: x86 + + - template: /.pipelines/templates/windows-package-build.yml@self + parameters: + runtime: fxdependent + + - template: /.pipelines/templates/windows-package-build.yml@self + parameters: + runtime: fxdependentWinDesktop + + - template: /.pipelines/templates/windows-package-build.yml@self + parameters: + runtime: minsize + + - stage: linux_package + jobs: + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64' + signedDrop: 'drop_linux_sign_linux_x64' + packageType: deb + jobName: deb + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd_x64_mariner' + signedDrop: 'drop_linux_sign_linux_fxd_x64_mariner' + packageType: rpm-fxdependent #mariner-x64 + jobName: mariner_x64 + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd_arm64_mariner' + signedDrop: 'drop_linux_sign_linux_fxd_arm64_mariner' + packageType: rpm-fxdependent-arm64 #mariner-arm64 + jobName: mariner_arm64 + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64' + signedDrop: 'drop_linux_sign_linux_x64' + packageType: rpm + jobName: rpm + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_arm' + signedDrop: 'drop_linux_sign_linux_arm' + packageType: tar-arm + jobName: tar_arm + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_arm64' + signedDrop: 'drop_linux_sign_linux_arm64' + packageType: tar-arm64 + jobName: tar_arm64 + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64_alpine' + signedDrop: 'drop_linux_sign_linux_x64_alpine' + packageType: tar-alpine + jobName: tar_alpine + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd' + signedDrop: 'drop_linux_sign_linux_fxd' + packageType: fxdependent + jobName: fxdependent + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64' + signedDrop: 'drop_linux_sign_linux_x64' + packageType: tar + jobName: tar + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd_x64_alpine' + signedDrop: 'drop_linux_sign_linux_fxd_x64_alpine' + packageType: tar-alpine-fxdependent + jobName: tar_alpine_fxd + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64_minSize' + signedDrop: 'drop_linux_sign_linux_x64_minSize' + packageType: min-size + jobName: minSize + + - stage: nupkg + jobs: + - template: /.pipelines/templates/nupkg.yml@self + + - stage: upload + dependsOn: [mac_package, windows_package, linux_package, nupkg] + jobs: + - template: /.pipelines/templates/uploadToAzure.yml@self diff --git a/.pipelines/templates/SetVersionVariables.yml b/.pipelines/templates/SetVersionVariables.yml index f8fb68c10bd..70aeff5b97b 100644 --- a/.pipelines/templates/SetVersionVariables.yml +++ b/.pipelines/templates/SetVersionVariables.yml @@ -13,7 +13,7 @@ steps: downloadPath: '$(System.ArtifactsDirectory)' displayName: Download Build Info Json env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue - powershell: | $path = "./build.psm1" @@ -43,7 +43,7 @@ steps: } displayName: 'Set repo Root' env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue - powershell: | $createJson = ("${{ parameters.CreateJson }}" -ne "no") @@ -58,11 +58,11 @@ steps: Write-Host "##$vstsCommandString" displayName: 'Set ${{ parameters.ReleaseTagVarName }} and other version Variables' env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue - powershell: | Get-ChildItem -Path env: displayName: Capture environment condition: succeededOrFailed() env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue diff --git a/.pipelines/templates/insert-nuget-config-azfeed.yml b/.pipelines/templates/insert-nuget-config-azfeed.yml index 19884080e46..e82102cbd74 100644 --- a/.pipelines/templates/insert-nuget-config-azfeed.yml +++ b/.pipelines/templates/insert-nuget-config-azfeed.yml @@ -5,7 +5,7 @@ steps: - pwsh: | $configPath = "${env:NugetConfigDir}/nuget.config" Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedUserName) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}" + New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}" if(-not (Test-Path $configPath)) { throw "nuget.config is not created" @@ -20,7 +20,7 @@ steps: - pwsh: | $configPath = "${env:NugetConfigDir}/nuget.config" Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedUserName) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}" + New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}" if(-not (Test-Path $configPath)) { throw "nuget.config is not created" diff --git a/.pipelines/templates/linux-package-build.yml b/.pipelines/templates/linux-package-build.yml new file mode 100644 index 00000000000..34b85bbdb02 --- /dev/null +++ b/.pipelines/templates/linux-package-build.yml @@ -0,0 +1,182 @@ +parameters: + unsignedDrop: 'drop_linux_build_linux_x64' + signedeDrop: 'drop_linux_sign_linux_x64' + packageType: deb + jobName: 'deb' + +jobs: +- job: ${{ parameters.jobName }} + displayName: Package linux ${{ parameters.packageType }} + condition: succeeded() + pool: + type: linux + + variables: + - name: runCodesignValidationInjection + value: false + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_binskim_enabled + value: true + - name: PackageType + value: ${{ parameters.packageType }} + - name: signedDrop + value: ${{ parameters.signedDrop }} + - name: unsignedDrop + value: ${{ parameters.unsignedDrop }} + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)/PowerShell/.config/tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - template: SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: shouldSign.yml + + - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' + + - download: CoOrdinatedBuildPipeline + artifact: ${{ parameters.unsignedDrop }} + displayName: 'Download unsigned artifacts' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - download: CoOrdinatedBuildPipeline + artifact: ${{ parameters.signedDrop }} + displayName: 'Download signed artifacts' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - pwsh: | + Write-Verbose -Verbose "Unsigned artifacts" + Get-ChildItem "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/${{ parameters.unsignedDrop }}" -Recurse + + Write-Verbose -Verbose "Signed artifacts" + Get-ChildItem "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/${{ parameters.signedDrop }}" -Recurse + displayName: 'Capture Downloaded Artifacts' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - pwsh: | + $packageType = '$(PackageType)' + Write-Verbose -Verbose "packageType = $packageType" + + $signedDrop = '$(signedDrop)' + Write-Verbose -Verbose "signedDrop = $signedDrop" + + $unsignedDrop = '$(unsignedDrop)' + Write-Verbose -Verbose "unsignedDrop = $unsignedDrop" + + Write-Verbose -Message "Init..." -Verbose + + $repoRoot = "$env:REPOROOT" + Import-Module "$repoRoot/build.psm1" + Import-Module "$repoRoot/tools/packaging" + + Start-PSBootstrap -Package + + $psOptionsPath = "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/${unsignedDrop}/psoptions/psoptions.json" + + if (-not (Test-Path $psOptionsPath)) { + throw "psOptionsPath file not found at $psOptionsPath" + } + + Restore-PSOptions $psOptionsPath + Write-Verbose -Message "Restoring PSOptions from $psoptionsFilePath" -Verbose + Get-PSOptions | Write-Verbose -Verbose + + $signedFolder, $pkgFilter = switch ($packageType) { + 'tar-arm' { 'Signed-linux-arm', 'powershell*.tar.gz' } + 'tar-arm64' { 'Signed-linux-arm64', 'powershell*.tar.gz' } + 'tar-alpine' { 'Signed-linux-musl-x64', 'powershell*.tar.gz' } + 'fxdependent' { 'Signed-fxdependent', 'powershell*.tar.gz' } + 'tar' { 'Signed-linux-x64', 'powershell*.tar.gz' } + 'tar-alpine-fxdependent' { 'Signed-fxdependent-noopt-linux-musl-x64', 'powershell*.tar.gz' } + 'deb' { 'Signed-linux-x64', 'powershell*.deb' } + 'rpm-fxdependent' { 'Signed-fxdependent-linux-x64', 'powershell*.rpm' } + 'rpm-fxdependent-arm64' { 'Signed-fxdependent-linux-arm64', 'powershell*.rpm' } + 'rpm' { 'Signed-linux-x64', 'powershell*.rpm' } + 'min-size' { 'Signed-linux-x64', 'powershell*.tar.gz' } + } + + $signedFilesPath = "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/${signedDrop}/${signedFolder}" + Write-Verbose -Verbose "signedFilesPath: $signedFilesPath" + + Write-Verbose -Message "checking pwsh exists in $signedFilesPath" -Verbose + if (-not (Test-Path "$signedFilesPath/pwsh")) { + throw "pwsh not found in $signedFilesPath" + } + + $metadata = Get-Content "$repoRoot/tools/metadata.json" -Raw | ConvertFrom-Json + $LTS = $metadata.LTSRelease.Package + + if ($LTS) { + Write-Verbose -Message "LTS Release: $LTS" + } + + if (-not (Test-Path $(ob_outputDirectory))) { + New-Item -ItemType Directory -Path $(ob_outputDirectory) -Force + } + + $packageType = '$(PackageType)' + Write-Verbose -Verbose "packageType = $packageType" + + Start-PSPackage -Type $packageType -ReleaseTag $(ReleaseTagVar) -PackageBinPath $signedFilesPath + + $vstsCommandString = "vso[task.setvariable variable=PackageFilter]$pkgFilter" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: 'Package ${{ parameters.packageType}}' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - task: onebranch.pipeline.signing@1 + displayName: Sign deb and rpm packages + inputs: + command: 'sign' + signing_profile: CP-459159-pgpdetached + files_to_sign: '**/*.rpm;**/*.deb' + search_root: '$(Pipeline.Workspace)' + + - pwsh: | + $pkgFilter = '$(PackageFilter)' + Write-Verbose -Verbose "pkgFilter: $pkgFilter" + + $pkgPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter $pkgFilter -Recurse -File | Select-Object -ExpandProperty FullName + Write-Verbose -Verbose "pkgPath: $pkgPath" + Copy-Item -Path $pkgPath -Destination '$(ob_outputDirectory)' -Force -Verbose + displayName: 'Copy artifacts to output directory' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + Get-ChildItem -Path $(ob_outputDirectory) -Recurse + displayName: 'List artifacts' diff --git a/.pipelines/templates/mac-package-build.yml b/.pipelines/templates/mac-package-build.yml new file mode 100644 index 00000000000..85f45e51a2c --- /dev/null +++ b/.pipelines/templates/mac-package-build.yml @@ -0,0 +1,129 @@ +parameters: + parentJob: '' + buildArchitecture: x64 + +jobs: +- job: package_macOS_${{ parameters.buildArchitecture }} + displayName: Package macOS ${{ parameters.buildArchitecture }} + condition: succeeded() + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-latest' + + variables: + - name: HOMEBREW_NO_ANALYTICS + value: 1 + - name: runCodesignValidationInjection + value: false + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_binskim_enabled + value: true + - name: ob_sdl_credscan_suppressionsfileforartifacts + value: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json + - name: BuildArch + value: ${{ parameters.buildArchitecture }} + + steps: + - checkout: self + clean: true + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment + + - pwsh: | + # create folder + sudo mkdir "$(Agent.TempDirectory)/PowerShell" + + # make the current user the owner + sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" + displayName: 'Create $(Agent.TempDirectory)/PowerShell' + + - template: SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: shouldSign.yml + + - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' + + - download: CoOrdinatedBuildPipeline + artifact: macosBinResults-${{ parameters.buildArchitecture }} + + - download: CoOrdinatedBuildPipeline + artifact: drop_macos_sign_${{ parameters.buildArchitecture }} + + - pwsh: | + Write-Verbose -Verbose "unsigned artifacts" + Get-ChildItem "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/macosBinResults-${{ parameters.buildArchitecture }}" -Recurse + + Write-Verbose -Verbose "unsigned artifacts" + Get-ChildItem "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/drop_macos_sign_${{ parameters.buildArchitecture }}" -Recurse + displayName: 'Capture Downloaded Artifacts' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + + - pwsh: | + # Add -SkipReleaseChecks as a mitigation to unblock release. + # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. + + $buildArch = '${{ parameters.buildArchitecture }}' + + Write-Verbose -Message "Init..." -Verbose + $repoRoot = $env:REPOROOT + Set-Location $repoRoot + Import-Module "$repoRoot/build.psm1" + Import-Module "$repoRoot/tools/packaging" + + $unsignedFilesPath = "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/macosBinResults-$buildArch" + $signedFilesPath = "$(Pipeline.Workspace)/CoOrdinatedBuildPipeline/drop_macos_sign_$buildArch/Signed-$buildArch" + + Write-Verbose -Message "checking pwsh exists in $signedFilesPath" -Verbose + if (-not (Test-Path $signedFilesPath/pwsh)) { + throw "pwsh not found in $signedFilesPath" + } + + $psoptionsPath = Get-ChildItem -Path $unsignedFilesPath -Filter 'psoptions.json' -Recurse -File | Select-Object -ExpandProperty FullName + Write-Verbose -Message "Restoring PSOptions from $psoptionsPath" -Verbose + + Restore-PSOptions -PSOptionsPath "$psoptionsPath" + Get-PSOptions | Write-Verbose -Verbose + + $metadata = Get-Content "$repoRoot/tools/metadata.json" -Raw | ConvertFrom-Json + $LTS = $metadata.LTSRelease.Package + + if ($LTS) { + Write-Verbose -Message "LTS Release: $LTS" + } + + Start-PSBootstrap -Package + + $macosRuntime = "osx-$buildArch" + + Start-PSPackage -Type osxpkg -SkipReleaseChecks -MacOSRuntime $macosRuntime -ReleaseTag $(ReleaseTagVar) -PackageBinPath $signedFilesPath -LTS:$LTS + $pkgNameFilter = "powershell-*$macosRuntime.pkg" + $pkgPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter $pkgNameFilter -Recurse -File | Select-Object -ExpandProperty FullName + Write-Host "##vso[artifact.upload containerfolder=macos-pkgs;artifactname=macos-pkgs]$pkgPath" + + Start-PSPackage -Type tar -SkipReleaseChecks -MacOSRuntime $macosRuntime -ReleaseTag $(ReleaseTagVar) -PackageBinPath $signedFilesPath -LTS:$LTS + $tarPkgNameFilter = "powershell-*$macosRuntime.tar.gz" + $tarPkgPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter $tarPkgNameFilter -Recurse -File | Select-Object -ExpandProperty FullName + Write-Host "##vso[artifact.upload containerfolder=macos-pkgs;artifactname=macos-pkgs]$tarPkgPath" + + displayName: 'Package ${{ parameters.buildArchitecture}}' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) diff --git a/.pipelines/templates/nupkg.yml b/.pipelines/templates/nupkg.yml new file mode 100644 index 00000000000..427654e2516 --- /dev/null +++ b/.pipelines/templates/nupkg.yml @@ -0,0 +1,310 @@ +jobs: +- job: build_nupkg + displayName: Package NuPkgs + condition: succeeded() + pool: + type: windows + + variables: + - name: runCodesignValidationInjection + value: false + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)\ONEBRANCH_ARTIFACT' + - name: ob_sdl_binskim_enabled + value: true + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - group: 'AzDevOpsArtifacts' + - group: DotNetPrivateBuildAccess + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - template: SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: shouldSign.yml + + - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' + + - download: CoOrdinatedBuildPipeline + artifact: drop_windows_build_windows_fxdependent_release + displayName: 'Download drop_windows_build_windows_fxdependent_release' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - download: CoOrdinatedBuildPipeline + artifact: drop_windows_build_windows_fxdependentWinDesktop_release + displayName: 'Download drop_windows_build_windows_fxdependentWinDesktop_release' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - download: CoOrdinatedBuildPipeline + artifact: drop_linux_sign_linux_fxd + displayName: 'Download drop_linux_sign_linux_fxd' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - download: CoOrdinatedBuildPipeline + artifact: drop_linux_sign_linux_fxd_x64_alpine + displayName: 'Download drop_linux_sign_linux_fxd_x64_alpine' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - pwsh: | + Write-Verbose -Verbose "drop_windows_build_windows_fxdependent_release" + Get-ChildItem -Path $(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependent_release -Recurse | Out-String | Write-Verbose -Verbose + + Write-Verbose -Verbose "drop_windows_build_windows_fxdependentWinDesktop_release" + Get-ChildItem -Path $(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependentWinDesktop_release -Recurse | Out-String | Write-Verbose -Verbose + + Write-Verbose -Verbose "drop_linux_sign_linux_fxd" + Get-ChildItem -Path $(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd -Recurse | Out-String | Write-Verbose -Verbose + + Write-Verbose -Verbose "drop_linux_sign_linux_fxd_x64_alpine" + Get-ChildItem -Path $(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd_x64_alpine -Recurse | Out-String | Write-Verbose -Verbose + displayName: 'Capture download artifacts' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - pwsh: | + $repoRoot = "$(PowerShellRoot)" + Write-Verbose -Verbose "repoRoot: $repoRoot" + + $configPath = "$repoRoot/nuget.config" + Import-Module "$repoRoot/build.psm1" -Force + New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)" + + if(-not (Test-Path $configPath)) + { + throw "nuget.config is not created" + } + Get-Content $configPath | Write-Verbose -Verbose + displayName: 'Add nuget.config for Azure DevOps feed for packages' + condition: and(succeededOrFailed(), ne(variables['PowerShellCore_PublicPackages'], '')) + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + - pwsh: | + Set-Location -Path '$(PowerShellRoot)' + Import-Module "$(PowerShellRoot)/build.psm1" -Force + + Start-PSBootstrap -Verbose + + $sharedModules = @('Microsoft.PowerShell.Commands.Management', + 'Microsoft.PowerShell.Commands.Utility', + 'Microsoft.PowerShell.ConsoleHost', + 'Microsoft.PowerShell.Security', + 'System.Management.Automation' + ) + + $winOnlyModules = @('Microsoft.Management.Infrastructure.CimCmdlets', + 'Microsoft.PowerShell.Commands.Diagnostics', + 'Microsoft.PowerShell.CoreCLR.Eventing', + 'Microsoft.WSMan.Management', + 'Microsoft.WSMan.Runtime' + ) + + $refAssemblyFolder = Join-Path '$(System.ArtifactsDirectory)' 'RefAssembly' + $null = New-Item -Path $refAssemblyFolder -Force -Verbose -Type Directory + + Start-PSBuild -Clean -Runtime linux-x64 -Configuration Release + + $sharedModules | Foreach-Object { + $refFile = Get-ChildItem -Path "$(PowerShellRoot)\src\$_\obj\Release\net9.0\refint\$_.dll" + Write-Verbose -Verbose "RefAssembly: $refFile" + Copy-Item -Path $refFile -Destination "$refAssemblyFolder\$_.dll" -Verbose + $refDoc = "$(PowerShellRoot)\src\$_\bin\Release\net9.0\$_.xml" + if (-not (Test-Path $refDoc)) { + Write-Warning "$refDoc not found" + Get-ChildItem -Path "$(PowerShellRoot)\src\$_\bin\Release\net9.0\" | Out-String | Write-Verbose -Verbose + } + else { + Copy-Item -Path $refDoc -Destination "$refAssemblyFolder\$_.xml" -Verbose + } + } + + Start-PSBuild -Clean -Runtime win7-x64 -Configuration Release + + $winOnlyModules | Foreach-Object { + $refFile = Get-ChildItem -Path "$(PowerShellRoot)\src\$_\obj\Release\net9.0\refint\*.dll" + Write-Verbose -Verbose 'RefAssembly: $refFile' + Copy-Item -Path $refFile -Destination "$refAssemblyFolder\$_.dll" -Verbose + $refDoc = "$(PowerShellRoot)\src\$_\bin\Release\net9.0\$_.xml" + if (-not (Test-Path $refDoc)) { + Write-Warning "$refDoc not found" + Get-ChildItem -Path "$(PowerShellRoot)\src\$_\bin\Release\net9.0" | Out-String | Write-Verbose -Verbose + } + else { + Copy-Item -Path $refDoc -Destination "$refAssemblyFolder\$_.xml" -Verbose + } + } + + Get-ChildItem $refAssemblyFolder -Recurse | Out-String | Write-Verbose -Verbose + + # Set RefAssemblyPath path variable + $vstsCommandString = "vso[task.setvariable variable=RefAssemblyPath]${refAssemblyFolder}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Build reference assemblies + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - task: onebranch.pipeline.signing@1 + displayName: Sign ref assemblies + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '**\*.dll' + search_root: '$(System.ArtifactsDirectory)\RefAssembly' + + - pwsh: | + $files = @( + "Microsoft.Management.Infrastructure.CimCmdlets.dll" + "Microsoft.PowerShell.Commands.Diagnostics.dll" + "Microsoft.PowerShell.Commands.Management.dll" + "Microsoft.PowerShell.Commands.Utility.dll" + "Microsoft.PowerShell.ConsoleHost.dll" + "Microsoft.PowerShell.CoreCLR.Eventing.dll" + "Microsoft.PowerShell.Security.dll" + "Microsoft.PowerShell.SDK.dll" + "Microsoft.WSMan.Management.dll" + "Microsoft.WSMan.Runtime.dll" + "System.Management.Automation.dll" + ) + + Import-Module -Name '$(PowerShellRoot)\build.psm1' + Import-Module -Name '$(PowerShellRoot)\tools\packaging' + Find-DotNet + + Write-Verbose -Verbose "Version == $(Version)" + + $winFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependent_release\Signed-fxdependent" + Write-Verbose -Verbose "winFxdPath == $winFxdPath" + + $linuxFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd\Signed-fxdependent" + Write-Verbose -Verbose "linuxFxdPath == $linuxFxdPath" + + $nupkgOutputPath = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'nupkg' + New-Item -Path $nupkgOutputPath -ItemType Directory -Force + + $files | Foreach-Object { + $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($_) + $FilePackagePath = Join-Path -Path $nupkgOutputPath -ChildPath $FileBaseName + Write-Verbose -Verbose "FileName to package: $_" + Write-Verbose -Verbose "FilePackage path: $FilePackagePath" + New-ILNugetPackageSource -File $_ -PackagePath $FilePackagePath -PackageVersion '$(Version)' -WinFxdBinPath $winFxdPath -LinuxFxdBinPath $linuxFxdPath -RefAssemblyPath $(RefAssemblyPath) + New-ILNugetPackageFromSource -FileName $_ -PackageVersion '$(Version)' -PackagePath $FilePackagePath + } + displayName: 'Create NuGet Package for single file' + + - task: onebranch.pipeline.signing@1 + displayName: Sign nupkg files + inputs: + command: 'sign' + cp_code: 'CP-401405' + files_to_sign: '**\*.nupkg' + search_root: '$(Pipeline.Workspace)\nupkg' + + ### Create global tools + + - pwsh: | + $winFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependent_release\Signed-fxdependent" + $winDesktopFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependentWinDesktop_release\Signed-fxdependent-win-desktop" + $linuxFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd\Signed-fxdependent" + $alpineFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd_x64_alpine\Signed-fxdependent-noopt-linux-musl-x64" + + Import-Module -Name '$(PowerShellRoot)\build.psm1' + Import-Module -Name '$(PowerShellRoot)\tools\packaging' + + Start-PrepForGlobalToolNupkg -LinuxBinPath $linuxFxdPath -WindowsBinPath $winFxdPath -WindowsDesktopBinPath $winDesktopFxdPath -AlpineBinPath $alpineFxdPath + displayName: 'Prepare for global tool packages' + + - pwsh: | + Import-Module -Name '$(PowerShellRoot)\build.psm1' + Import-Module -Name '$(PowerShellRoot)\tools\packaging' + Find-DotNet + + $gblToolOutputPath = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'globaltools' + New-Item -Path $gblToolOutputPath -ItemType Directory -Force + + $winFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependent_release\Signed-fxdependent" + $winDesktopFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_fxdependentWinDesktop_release\Signed-fxdependent-win-desktop" + $linuxFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd\Signed-fxdependent" + $alpineFxdPath = "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_linux_sign_linux_fxd_x64_alpine\Signed-fxdependent-noopt-linux-musl-x64" + + $packageTypes = @('Unified', 'PowerShell.Linux.Alpine', 'PowerShell.Linux.x64', 'PowerShell.Linux.arm32', 'PowerShell.Linux.arm64', 'PowerShell.Windows.x64') + + $packageTypes | Foreach-Object { + $PackageType = $_ + Write-Verbose -Verbose "PackageType: $PackageType" + + New-GlobalToolNupkgSource -PackageType $PackageType -PackageVersion '$(Version)' -LinuxBinPath $linuxFxdPath -WindowsBinPath $winFxdPath -WindowsDesktopBinPath $winDesktopFxdPath -AlpineBinPath $alpineFxdPath -SkipCGManifest + + Write-Verbose -Verbose "GlobalToolNuspecSourcePath = $global:GlobalToolNuSpecSourcePath" + Write-Verbose -Verbose "GlobalToolPkgName = $global:GlobalToolPkgName" + + Write-Verbose -Verbose "Starting global tool package creation for $PackageType" + New-GlobalToolNupkgFromSource -PackageNuSpecPath "$global:GlobalToolNuSpecSourcePath" -PackageName "$global:GlobalToolPkgName" -DestinationPath $gblToolOutputPath + Write-Verbose -Verbose "Global tool package created for $PackageType" + $global:GlobalToolNuSpecSourcePath = $null + $global:GlobalToolPkgName = $null + } + displayName: 'Create global tools' + + - pwsh: | + $gblToolOutputPath = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'globaltools' + Get-ChildItem -Path $gblToolOutputPath + displayName: Capture global tools + + - task: onebranch.pipeline.signing@1 + displayName: Sign nupkg files + inputs: + command: 'sign' + cp_code: 'CP-401405' + files_to_sign: '**\*.nupkg' + search_root: '$(Pipeline.Workspace)\globaltools' + + - pwsh: | + if (-not (Test-Path '$(ob_outputDirectory)')) { + New-Item -ItemType Directory -Path '$(ob_outputDirectory)' -Force + } + + Write-Verbose -Verbose "Copying nupkgs to output directory" + $nupkgOutputPath = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'nupkg' + Get-ChildItem -Path $nupkgOutputPath -Filter *.nupkg -Recurse | Copy-Item -Destination '$(ob_outputDirectory)' -Force -Verbose + + Write-Verbose -Verbose "Copying global tools to output directory" + $gblToolOutputPath = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'globaltools' + Get-ChildItem -Path $gblToolOutputPath -Filter *.nupkg -Recurse | Copy-Item -Destination '$(ob_outputDirectory)' -Force -Verbose + displayName: Copy artifacts to output directory + + - pwsh: | + $nupkgOutputPath = '$(ob_outputDirectory)' + Get-ChildItem -Path $nupkgOutputPath | Out-String | Write-Verbose -Verbose + displayName: List artifacts diff --git a/.pipelines/templates/shouldSign.yml b/.pipelines/templates/shouldSign.yml index 3f024898945..4bac9e1a3ae 100644 --- a/.pipelines/templates/shouldSign.yml +++ b/.pipelines/templates/shouldSign.yml @@ -21,3 +21,5 @@ steps: Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" displayName: 'Set SHOULD_SIGN Variable' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml new file mode 100644 index 00000000000..00fd3286749 --- /dev/null +++ b/.pipelines/templates/uploadToAzure.yml @@ -0,0 +1,377 @@ +jobs: +- job: upload_packages + displayName: Upload packages + condition: succeeded() + pool: + type: windows + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: + - name: ob_sdl_codeql_compiled_enabled + value: true + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: /.pipelines/templates/cloneToOfficialPath.yml@self + + - pwsh: | + Get-ChildItem Env: + displayName: 'Capture Environment Variables' + + - pwsh: | + New-Item -Path '$(Build.ArtifactStagingDirectory)/downloads' -ItemType Directory -Force + displayName: Create downloads directory + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_deb + itemPattern: '**/*.deb' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download deb package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_fxdependent + itemPattern: '**/*.tar.gz' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux fxd package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_mariner_arm64 + itemPattern: '**/*.rpm' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux mariner arm64 package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_mariner_x64 + itemPattern: '**/*.rpm' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux mariner x64 package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_minSize + itemPattern: '**/*.tar.gz' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux minSize package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_rpm + itemPattern: '**/*.rpm' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux rpm package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_tar + itemPattern: '**/*.tar.gz' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux tar package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_tar_alpine + itemPattern: '**/*.tar.gz' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux alpine tar package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_tar_alpine_fxd + itemPattern: '**/*.tar.gz' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux alpine fxd tar package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_tar_arm + itemPattern: '**/*.tar.gz' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux arm32 tar package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_linux_package_tar_arm64 + itemPattern: '**/*.tar.gz' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download linux arm64 tar package + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_nupkg_build_nupkg + itemPattern: '**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download nupkgs + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_windows_package_package_win_arm64 + itemPattern: | + **/*.msi + **/*.msix + **/*.zip + **/*.exe + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download windows arm64 packages + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_windows_package_package_win_fxdependent + itemPattern: '**/*.zip' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download windows fxdependent packages + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_windows_package_package_win_fxdependentWinDesktop + itemPattern: '**/*.zip' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download windows fxdependentWinDesktop packages + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_windows_package_package_win_minsize + itemPattern: '**/*.zip' + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download windows minsize packages + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_windows_package_package_win_x64 + itemPattern: | + **/*.msi + **/*.msix + **/*.zip + **/*.exe + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download windows x64 packages + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_windows_package_package_win_x86 + itemPattern: | + **/*.msi + **/*.msix + **/*.zip + **/*.exe + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download windows x86 packages + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: macos-pkgs + itemPattern: | + **/*.pkg + **/*.tar.gz + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download macos packages + + - pwsh: | + Get-ChildItem '$(Build.ArtifactStagingDirectory)/downloads' | Select-Object -ExpandProperty FullName + displayName: 'Capture downloads' + + - pwsh: | + # Create output directory for packages which have been uploaded to blob storage + New-Item -Path $(Build.ArtifactStagingDirectory)/uploaded -ItemType Directory -Force + displayName: Create output directory for packages + + - pwsh: | + $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose + if ($azureRmModule) { + Write-Host 'AzureRM module exists. Removing it' + Uninstall-AzureRm + Write-Host 'AzureRM module removed' + } + + Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + + displayName: Remove AzRM modules + + - task: AzurePowerShell@5 + displayName: Upload packages to blob + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + pwsh: true + inline: | + $downloadsDirectory = '$(Build.ArtifactStagingDirectory)/downloads' + $uploadedDirectory = '$(Build.ArtifactStagingDirectory)/uploaded' + $storageAccountName = "pscoretestdata" + $containerName = $env:AZUREVERSION + + Write-Verbose -Verbose "Uploading packages to blob storage account: $storageAccountName container: $containerName" + + $context = New-AzStorageContext -StorageAccountName $storageAccountName -UseConnectedAccount + + # Create the blob container if it doesn't exist + $containerExists = Get-AzStorageContainer -Name $containerName -Context $context -ErrorAction SilentlyContinue + if (-not $containerExists) { + $null = New-AzStorageContainer -Name $containerName -Context $context + Write-Host "Blob container $containerName created successfully." + } + + $gcPackages = Get-ChildItem -Path $downloadsDirectory -Filter "powershell*gc.*" + Write-Verbose -Verbose "gc files to upload." + $gcPackages | Write-Verbose -Verbose + $gcContainerName = "$containerName-gc" + # Create the blob container if it doesn't exist + $containerExists = Get-AzStorageContainer -Name $gcContainerName -Context $context -ErrorAction SilentlyContinue + if (-not $containerExists) { + $null = New-AzStorageContainer -Name $gcContainerName -Context $context + Write-Host "Blob container $gcContainerName created successfully." + } + + $gcPackages | ForEach-Object { + $blobName = "${_.Name}" + Write-Verbose -Verbose "Uploading $($_.FullName) to $gcContainerName/$blobName" + $null = Set-AzStorageBlobContent -File $_.FullName -Container $gcContainerName -Blob $blobName -Context $context + # Move to folder to we wont upload again + Move-Item -Path $_.FullName -Destination $uploadedDirectory -Force -Verbose + } + + $nupkgFiles = Get-ChildItem -Path $downloadsDirectory -Filter "*.nupkg" | Where-Object { $_.Name -notlike "powershell*.nupkg" } + + # create a SHA512 checksum file for each nupkg files + + $checksums = $nupkgFiles | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + + $checksums | Out-File -FilePath "$downloadsDirectory\SHA512SUMS" -Force + $fileContent = Get-Content -Path "$downloadsDirectory\SHA512SUMS" -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent + + Write-Verbose -Verbose "nupkg files to upload." + $nupkgFiles += (Get-Item "$downloadsDirectory\SHA512SUMS") + $nupkgFiles | Write-Verbose -Verbose + $nugetContainerName = "$containerName-nuget" + # Create the blob container if it doesn't exist + $containerExists = Get-AzStorageContainer -Name $nugetContainerName -Context $context -ErrorAction SilentlyContinue + if (-not $containerExists) { + $null = New-AzStorageContainer -Name $nugetContainerName -Context $context + Write-Host "Blob container $nugetContainerName created successfully." + } + + $nupkgFiles | ForEach-Object { + $blobName = $_.Name + Write-Verbose -Verbose "Uploading $($_.FullName) to $nugetContainerName/$blobName" + $null = Set-AzStorageBlobContent -File $_.FullName -Container $nugetContainerName -Blob $blobName -Context $context + # Move to folder to we wont upload again + Move-Item -Path $_.FullName -Destination $uploadedDirectory -Force -Verbose + } + + $globaltoolFiles = Get-ChildItem -Path $downloadsDirectory -Filter "powershell*.nupkg" + # create a SHA512 checksum file for each nupkg files + + $checksums = $globaltoolFiles | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + + New-Item -Path "$downloadsDirectory\globaltool" -ItemType Directory -Force + $checksums | Out-File -FilePath "$downloadsDirectory\globaltool\SHA512SUMS" -Force + $fileContent = Get-Content -Path "$downloadsDirectory\globaltool\SHA512SUMS" -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent + + Write-Verbose -Verbose "globaltool files to upload." + $globaltoolFiles += Get-Item ("$downloadsDirectory\globaltool\SHA512SUMS") + $globaltoolFiles | Write-Verbose -Verbose + $globaltoolContainerName = "$containerName-nuget" + $globaltoolFiles | ForEach-Object { + $blobName = "globaltool/" + $_.Name + $globaltoolContainerName = "$containerName-nuget" + Write-Verbose -Verbose "Uploading $($_.FullName) to $globaltoolContainerName/$blobName" + $null = Set-AzStorageBlobContent -File $_.FullName -Container $globaltoolContainerName -Blob $blobName -Context $context + # Move to folder to we wont upload again + Move-Item -Path $_.FullName -Destination $uploadedDirectory -Force + } + + # To use -Include parameter, we need to use \* to get all files + $privateFiles = Get-ChildItem -Path $downloadsDirectory\* -Include @("*.msix", "*.exe") + Write-Verbose -Verbose "private files to upload." + $privateFiles | Write-Verbose -Verbose + $privateContainerName = "$containerName-private" + # Create the blob container if it doesn't exist + $containerExists = Get-AzStorageContainer -Name $privateContainerName -Context $context -ErrorAction SilentlyContinue + if (-not $containerExists) { + $null = New-AzStorageContainer -Name $privateContainerName -Context $context + Write-Host "Blob container $privateContainerName created successfully." + } + + $privateFiles | ForEach-Object { + $blobName = $_.Name + Write-Verbose -Verbose "Uploading $($_.FullName) to $privateContainerName/$blobName" + $null = Set-AzStorageBlobContent -File $_.FullName -Container $privateContainerName -Blob $blobName -Context $context + # Move to folder to we wont upload again + Move-Item -Path $_.FullName -Destination $uploadedDirectory -Force -Verbose + } + + # To use -Include parameter, we need to use \* to get all files + $files = Get-ChildItem -Path $downloadsDirectory\* -Include @("*.deb", "*.tar.gz", "*.rpm", "*.msi", "*.zip", "*.pkg") + Write-Verbose -Verbose "files to upload." + $files | Write-Verbose -Verbose + + $files | ForEach-Object { + $blobName = $_.Name + Write-Verbose -Verbose "Uploading $($_.FullName) to $containerName/$blobName" + $null = Set-AzStorageBlobContent -File $_.FullName -Container $containerName -Blob $blobName -Context $context + Write-Host "File $blobName uploaded to $containerName container." + Move-Item -Path $_.FullName -Destination $uploadedDirectory -Force -Verbose + } diff --git a/.pipelines/templates/windows-package-build.yml b/.pipelines/templates/windows-package-build.yml new file mode 100644 index 00000000000..9861c2a7314 --- /dev/null +++ b/.pipelines/templates/windows-package-build.yml @@ -0,0 +1,277 @@ +parameters: + runtime: x64 + +jobs: +- job: package_win_${{ parameters.runtime }} + displayName: Package Windows ${{ parameters.runtime }} + condition: succeeded() + pool: + type: windows + + variables: + - name: runCodesignValidationInjection + value: false + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)\ONEBRANCH_ARTIFACT' + - name: ob_sdl_binskim_enabled + value: true + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: Runtime + value: ${{ parameters.runtime }} + - group: msixTools + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture environment + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - template: SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + UseJson: no + + - template: shouldSign.yml + + - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' + + - download: CoOrdinatedBuildPipeline + artifact: drop_windows_build_windows_${{ parameters.runtime }}_release + displayName: Download signed artifacts + condition: ${{ ne(parameters.runtime, 'minSize') }} + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - download: CoOrdinatedBuildPipeline + artifact: drop_windows_build_windows_x64_${{ parameters.runtime }} + displayName: Download minsize signed artifacts + condition: ${{ eq(parameters.runtime, 'minSize') }} + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - pwsh: | + Write-Verbose -Verbose "signed artifacts" + Get-ChildItem "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_${{ parameters.runtime }}_release" -Recurse + displayName: 'Capture Downloaded Artifacts' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - pwsh: | + # cleanup previous install + if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { + Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force + } + $toolsDir = New-Item -ItemType Directory -Path '$(Build.ArtifactStagingDirectory)\tools' + $wixUri = 'https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314-binaries.zip' + + Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' -MaximumRetryCount 5 -RetryIntervalSec 10 + Import-Module '$(PowerShellRoot)\tools\releaseBuild\Images\microsoft_powershell_windowsservercore\wix.psm1' + $isArm64 = '$(Runtime)' -eq 'arm64' + + Install-WixZip -zipPath '$(Build.ArtifactStagingDirectory)\tools\wix.zip' -arm64:$isArm64 + + $msixUrl = '$(makeappUrl)' + Invoke-RestMethod -Uri $msixUrl -OutFile '$(Pipeline.Workspace)\makeappx.zip' + Expand-Archive '$(Pipeline.Workspace)\makeappx.zip' -destination '\' -Force + displayName: Install packaging tools + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - pwsh: | + $runtime = '$(Runtime)' + Write-Verbose -Verbose "runtime = '$(Runtime)'" + + $signedFolder = switch ($runtime) { + 'x64' { 'Signed-win7-x64' } + 'x86' { 'Signed-win7-x86' } + 'arm64' { 'Signed-win-arm64' } + 'fxdependent' { 'Signed-fxdependent' } + 'fxdependentWinDesktop' { 'Signed-fxdependent-win-desktop' } + 'minsize' { 'Signed-win7-x64' } + } + + Write-Verbose -Message "Init..." -Verbose + + $repoRoot = "$env:REPOROOT" + Import-Module "$repoRoot\build.psm1" + Import-Module "$repoRoot\tools\packaging" + + Start-PSBootstrap -Package + + $signedFilesPath, $psoptionsFilePath = if ($env:RUNTIME -eq 'minsize') { + "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_x64_${runtime}\$signedFolder" + "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_x64_${runtime}\psoptions\psoptions.json" + } + else { + "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_${runtime}_release\$signedFolder" + "$(Pipeline.Workspace)\CoOrdinatedBuildPipeline\drop_windows_build_windows_${runtime}_release\psoptions\psoptions.json" + } + + Write-Verbose -Verbose "signedFilesPath: $signedFilesPath" + Write-Verbose -Verbose "psoptionsFilePath: $psoptionsFilePath" + + Write-Verbose -Message "checking pwsh exists in $signedFilesPath" -Verbose + if (-not (Test-Path $signedFilesPath\pwsh.exe)) { + throw "pwsh.exe not found in $signedFilesPath" + } + + Write-Verbose -Message "Restoring PSOptions from $psoptionsFilePath" -Verbose + + Restore-PSOptions -PSOptionsPath "$psoptionsFilePath" + Get-PSOptions | Write-Verbose -Verbose + + $metadata = Get-Content "$repoRoot/tools/metadata.json" -Raw | ConvertFrom-Json + $LTS = $metadata.LTSRelease.Package + + if ($LTS) { + Write-Verbose -Message "LTS Release: $LTS" + } + + Start-PSBootstrap -Package + + $WindowsRuntime = switch ($runtime) { + 'x64' { 'win7-x64' } + 'x86' { 'win7-x86' } + 'arm64' { 'win-arm64' } + 'fxdependent' { 'win7-x64' } + 'fxdependentWinDesktop' { 'win7-x64' } + 'minsize' { 'win7-x64' } + } + + $packageTypes = switch ($runtime) { + 'x64' { @('msi', 'zip', 'msix') } + 'x86' { @('msi', 'zip', 'msix') } + 'arm64' { @('msi', 'zip', 'msix') } + 'fxdependent' { 'fxdependent' } + 'fxdependentWinDesktop' { 'fxdependent-win-desktop' } + 'minsize' { 'min-size' } + } + + if (-not (Test-Path $(ob_outputDirectory))) { + New-Item -ItemType Directory -Path $(ob_outputDirectory) -Force + } + + Set-Location $repoRoot + + Start-PSPackage -Type $packageTypes -SkipReleaseChecks -WindowsRuntime $WindowsRuntime -ReleaseTag $(ReleaseTagVar) -PackageBinPath $signedFilesPath -LTS:$LTS + + displayName: 'Package ${{ parameters.buildArchitecture}}' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + + - task: onebranch.pipeline.signing@1 + displayName: Sign MSI packages + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '**\*.msi' + search_root: '$(Pipeline.Workspace)' + + - pwsh: | + $runtime = '$(Runtime)' + Write-Verbose -Verbose "runtime = '$(Runtime)'" + + $repoRoot = "$env:REPOROOT" + Import-Module "$repoRoot\build.psm1" + Import-Module "$repoRoot\tools\packaging" + + $noExeRuntimes = @('fxdependent', 'fxdependentWinDesktop', 'minsize') + + if ($runtime -in $noExeRuntimes) { + Write-Verbose -Verbose "No EXE generated for $runtime" + return + } + + $version = '$(Version)' + + $msiLocation = Get-ChildItem -Path $(Pipeline.Workspace) -Recurse -Filter "powershell-*$runtime.msi" | Select-Object -ExpandProperty FullName + Write-Verbose -Verbose "msiLocation: $msiLocation" + + Set-Location $repoRoot + + $exePath = New-ExePackage -ProductVersion $version -ProductTargetArchitecture $runtime -MsiLocationPath $msiLocation + Write-Verbose -Verbose "exePath: $exePath" + displayName: 'Make exe package' + + - task: onebranch.pipeline.signing@1 + displayName: Sign MSI packages + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '**\*.exe' + search_root: '$(Pipeline.Workspace)' + + - pwsh: | + $runtime = '$(Runtime)' + Write-Verbose -Verbose "runtime = '$(Runtime)'" + + $packageTypes = switch ($runtime) { + 'x64' { @('msi', 'zip', 'msix', 'exe') } + 'x86' { @('msi', 'zip', 'msix', 'exe') } + 'arm64' { @('msi', 'zip', 'msix', 'exe') } + 'fxdependent' { 'fxdependent' } + 'fxdependentWinDesktop' { 'fxdependent-win-desktop' } + 'minsize' { 'min-size' } + } + + if (-not (Test-Path $(ob_outputDirectory))) { + New-Item -ItemType Directory -Path $(ob_outputDirectory) -Force + } + + if ($packageTypes -contains 'msi') { + $msiPkgNameFilter = "powershell-*.msi" + $msiPkgPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter $msiPkgNameFilter -Recurse -File | Select-Object -ExpandProperty FullName + Write-Verbose -Verbose "msiPkgPath: $msiPkgPath" + Copy-Item -Path $msiPkgPath -Destination '$(ob_outputDirectory)' -Force -Verbose + } + + if ($packageTypes -contains 'exe') { + $msiPkgNameFilter = "powershell-*.exe" + $msiPkgPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter $msiPkgNameFilter -Recurse -File | Select-Object -ExpandProperty FullName + Write-Verbose -Verbose "msiPkgPath: $msiPkgPath" + Copy-Item -Path $msiPkgPath -Destination '$(ob_outputDirectory)' -Force -Verbose + } + + if ($packageTypes -contains 'zip' -or $packageTypes -contains 'fxdependent' -or $packageTypes -contains 'min-size' -or $packageTypes -contains 'fxdependent-win-desktop') { + $zipPkgNameFilter = "powershell-*.zip" + $zipPkgPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter $zipPkgNameFilter -Recurse -File | Select-Object -ExpandProperty FullName + Write-Verbose -Verbose "zipPkgPath: $zipPkgPath" + Copy-Item -Path $zipPkgPath -Destination '$(ob_outputDirectory)' -Force -Verbose + } + + if ($packageTypes -contains 'msix') { + $msixPkgNameFilter = "powershell-*.msix" + $msixPkgPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter $msixPkgNameFilter -Recurse -File | Select-Object -ExpandProperty FullName + Write-Verbose -Verbose "msixPkgPath: $msixPkgPath" + Copy-Item -Path $msixPkgPath -Destination '$(ob_outputDirectory)' -Force -Verbose + } + displayName: Copy to output directory + + - pwsh: | + Get-ChildItem -Path $(ob_outputDirectory) -Recurse + displayName: 'List artifacts' + env: + ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue + diff --git a/build.psm1 b/build.psm1 index 162a5a5ef5b..1b169d62c99 100644 --- a/build.psm1 +++ b/build.psm1 @@ -191,6 +191,7 @@ function Get-EnvironmentInformation $environment += @{'IsRedHatFamily' = $environment.IsCentOS -or $environment.IsFedora -or $environment.IsRedHat} $environment += @{'IsSUSEFamily' = $environment.IsSLES -or $environment.IsOpenSUSE} $environment += @{'IsAlpine' = $LinuxInfo.ID -match 'alpine'} + $environment += @{'IsMariner' = $LinuxInfo.ID -match 'mariner'} # Workaround for temporary LD_LIBRARY_PATH hack for Fedora 24 # https://github.com/PowerShell/PowerShell/issues/2511 @@ -204,7 +205,8 @@ function Get-EnvironmentInformation $environment.IsUbuntu -or $environment.IsRedHatFamily -or $environment.IsSUSEFamily -or - $environment.IsAlpine) + $environment.IsAlpine -or + $environment.IsMariner) ) { if ($SkipLinuxDistroCheck) { Write-Warning "The current OS : $($LinuxInfo.ID) is not supported for building PowerShell." @@ -2168,7 +2170,7 @@ function Start-PSBootstrap { # change the apt frontend back to the original $env:DEBIAN_FRONTEND=$originalDebianFrontEnd } - } elseif ($environment.IsLinux -and $environment.IsRedHatFamily) { + } elseif ($environment.IsLinux -and ($environment.IsRedHatFamily -or $environment.IsMariner)) { # Build tools $Deps += "which", "curl", "wget" @@ -2239,9 +2241,10 @@ function Start-PSBootstrap { # Install [fpm](https://github.com/jordansissel/fpm) and [ronn](https://github.com/rtomayko/ronn) if ($Package) { Install-GlobalGem -Sudo $sudo -GemName "dotenv" -GemVersion "2.8.1" - Install-GlobalGem -Sudo $sudo -GemName "ffi" -GemVersion "1.12.0" - Install-GlobalGem -Sudo $sudo -GemName "fpm" -GemVersion "1.11.0" + Install-GlobalGem -Sudo $sudo -GemName "ffi" -GemVersion "1.16.3" + Install-GlobalGem -Sudo $sudo -GemName "fpm" -GemVersion "1.15.1" Install-GlobalGem -Sudo $sudo -GemName "ronn" -GemVersion "0.7.3" + Install-GlobalGem -Sudo $sudo -GemName "rexml" -GemVersion "3.2.5" } } diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index a6575088090..f7295bffcf1 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -61,6 +61,8 @@ function Start-PSPackage { [ValidateScript({$Environment.IsMacOS})] [string] $MacOSRuntime, + [string] $PackageBinPath, + [switch] $Private, [Switch] $Force, @@ -245,7 +247,14 @@ function Start-PSPackage { $Version = (git --git-dir="$RepoRoot/.git" describe) -Replace '^v' } - $Source = Split-Path -Path $Script:Options.Output -Parent + $Source = if ($PackageBinPath) { + $PackageBinPath + } + else { + Split-Path -Path $Script:Options.Output -Parent + } + + Write-Verbose -Verbose "Source: $Source" # Copy the ThirdPartyNotices.txt so it's part of the package Copy-Item "$RepoRoot/ThirdPartyNotices.txt" -Destination $Source -Force @@ -890,12 +899,63 @@ function Update-PSSignedBuildFolder ) # Replace unsigned binaries with signed - $signedFilesFilter = Join-Path -Path $SignedFilesPath -ChildPath '*' - Get-ChildItem -Path $signedFilesFilter -Recurse -File | Select-Object -ExpandProperty FullName | ForEach-Object -Process { - $relativePath = $_.ToLowerInvariant().Replace($SignedFilesPath.ToLowerInvariant(),'') - $destination = Join-Path -Path $BuildPath -ChildPath $relativePath - Write-Log "replacing $destination with $_" - Copy-Item -Path $_ -Destination $destination -Force + $signedFilesFilter = Join-Path -Path $SignedFilesPathNormalized -ChildPath '*' + Write-Verbose -Verbose "signedFilesFilter = $signedFilesFilter" + + $signedFilesList = Get-ChildItem -Path $signedFilesFilter -Recurse -File + foreach ($signedFileObject in $signedFilesList) { + # completely skip replacing pwsh on non-windows systems (there is no .exe extension here) + # and it may not be signed correctly + + # The Shim will not be signed in CI. + + if ($signedFileObject.Name -eq "pwsh" -or ($signedFileObject.Name -eq "Microsoft.PowerShell.GlobalTool.Shim.exe" -and $env:BUILD_REASON -eq 'PullRequest')) { + Write-Verbose -Verbose "Skipping $signedFileObject" + continue + } + + $signedFilePath = $signedFileObject.FullName + Write-Verbose -Verbose "Processing $signedFilePath" + + # Agents seems to be on a case sensitive file system + if ($IsLinux) { + $relativePath = $signedFilePath.Replace($SignedFilesPathNormalized, '') + } else { + $relativePath = $signedFilePath.ToLowerInvariant().Replace($SignedFilesPathNormalized.ToLowerInvariant(), '') + } + + Write-Verbose -Verbose "relativePath = $relativePath" + $destination = (Get-Item (Join-Path -Path $BuildPathNormalized -ChildPath $relativePath)).FullName + Write-Verbose -Verbose "destination = $destination" + Write-Log "replacing $destination with $signedFilePath" + + if (-not (Test-Path $destination)) { + $parent = Split-Path -Path $destination -Parent + $exists = Test-Path -Path $parent + + if ($exists) { + Write-Verbose -Verbose "Parent:" + Get-ChildItem -Path $parent | Select-Object -ExpandProperty FullName | Write-Verbose -Verbose + } + + Write-Error "File not found: $destination, parent - $parent exists: $exists" + } + + # Get-AuthenticodeSignature will only work on Windows + if ($IsWindows) + { + $signature = Get-AuthenticodeSignature -FilePath $signedFilePath + if ($signature.Status -ne 'Valid') { + Write-Error "Invalid signature for $signedFilePath" + } + } + else + { + Write-Verbose -Verbose "Skipping certificate check of $signedFilePath on non-Windows" + } + + Copy-Item -Path $signedFilePath -Destination $destination -Force + } foreach($filter in $RemoveFilter) { @@ -1027,7 +1087,7 @@ function New-UnixPackage { switch ($Type) { "deb" { $packageVersion = Get-LinuxPackageSemanticVersion -Version $Version - if (!$Environment.IsUbuntu -and !$Environment.IsDebian) { + if (!$Environment.IsUbuntu -and !$Environment.IsDebian -and !$Environment.IsMariner) { throw ($ErrorMessage -f "Ubuntu or Debian") } @@ -1626,7 +1686,7 @@ function New-AfterScripts $packagingStrings.RedHatAfterInstallScript -f "$Link", $Destination | Out-File -FilePath $AfterInstallScript -Encoding ascii $packagingStrings.RedHatAfterRemoveScript -f "$Link", $Destination | Out-File -FilePath $AfterRemoveScript -Encoding ascii } - elseif ($Environment.IsDebianFamily -or $Environment.IsSUSEFamily) { + elseif ($Environment.IsDebianFamily -or $Environment.IsSUSEFamily -or $Distribution -in $script:DebianDistributions) { $AfterInstallScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) $AfterRemoveScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) $packagingStrings.UbuntuAfterInstallScript -f "$Link", $Destination | Out-File -FilePath $AfterInstallScript -Encoding ascii @@ -2238,7 +2298,6 @@ function New-ILNugetPackageSource [Parameter(Mandatory = $true)] [string] $RefAssemblyPath, - [Parameter(Mandatory = $true)] [string] $CGManifestPath ) @@ -2295,9 +2354,15 @@ function New-ILNugetPackageSource CreateNugetPlatformFolder -FileName $FileName -Platform 'win' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $WinFxdBinPath + Write-Verbose -Verbose "Done creating Windows runtime assemblies for $FileName" + if ($linuxExceptionList -notcontains $FileName ) { CreateNugetPlatformFolder -FileName $FileName -Platform 'unix' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $LinuxFxdBinPath + Write-Verbose -Verbose "Done creating Linux runtime assemblies for $FileName" + } + else { + Write-Verbose -Verbose "Skipping creating Linux runtime assemblies for $FileName" } if ($FileName -eq "Microsoft.PowerShell.SDK.dll") @@ -2346,6 +2411,14 @@ function New-ILNugetPackageSource Write-Log "Copied the built-in modules to contentFiles for the SDK package" } + else { + Write-Verbose -Verbose "Skipping copying the built-in modules and reference assemblies for $FileName" + } + + if (-not $PSBoundParameters.ContainsKey("CGManifestPath")) { + Write-Verbose -Verbose "CGManifestPath is not provided. Skipping CGManifest creation." + return + } # Create a CGManifest file that lists all dependencies for this package, which is used when creating the SBOM. if (! (Test-Path -Path $CGManifestPath)) { @@ -4119,7 +4192,8 @@ function New-GlobalToolNupkgSource [Parameter(Mandatory)] [string] $WindowsBinPath, [Parameter(Mandatory)] [string] $WindowsDesktopBinPath, [Parameter(Mandatory)] [string] $AlpineBinPath, - [Parameter(Mandatory)] [string] $PackageVersion + [Parameter(Mandatory)] [string] $PackageVersion, + [Parameter()] [switch] $SkipCGManifest ) if ($PackageType -ne "Unified") @@ -4283,12 +4357,21 @@ function New-GlobalToolNupkgSource # Set VSTS environment variable for package NuSpec source path. $pkgNuSpecSourcePathVar = "GlobalToolNuSpecSourcePath" Write-Log "New-GlobalToolNupkgSource: Creating NuSpec source path VSTS variable: $pkgNuSpecSourcePathVar" + Write-Verbose -Verbose "sending: [task.setvariable variable=$pkgNuSpecSourcePathVar]$RootFolder" Write-Host "##vso[task.setvariable variable=$pkgNuSpecSourcePathVar]$RootFolder" + $global:GlobalToolNuSpecSourcePath = $RootFolder # Set VSTS environment variable for package Name. $pkgNameVar = "GlobalToolPkgName" Write-Log "New-GlobalToolNupkgSource: Creating current package name variable: $pkgNameVar" + Write-Verbose -Verbose "sending: vso[task.setvariable variable=$pkgNameVar]$PackageName" Write-Host "##vso[task.setvariable variable=$pkgNameVar]$PackageName" + $global:GlobalToolPkgName = $PackageName + + if ($SkipCGManifest.IsPresent) { + Write-Verbose -Verbose "New-GlobalToolNupkgSource: Skipping CGManifest creation." + return + } # Set VSTS environment variable for CGManifest file path. $globalToolCGManifestPFilePath = Join-Path -Path "$env:REPOROOT" -ChildPath "tools\cgmanifest.json" @@ -4331,7 +4414,7 @@ function New-GlobalToolNupkgFromSource [Parameter(Mandatory)] [string] $PackageNuSpecPath, [Parameter(Mandatory)] [string] $PackageName, [Parameter(Mandatory)] [string] $DestinationPath, - [Parameter(Mandatory)] [string] $CGManifestPath + [Parameter()] [string] $CGManifestPath ) if (! (Test-Path -Path $PackageNuSpecPath)) @@ -4345,6 +4428,12 @@ function New-GlobalToolNupkgFromSource Write-Log "New-GlobalToolNupkgFromSource: Removing GlobalTool NuSpec source directory: $PackageNuSpecPath" Remove-Item -Path $PackageNuSpecPath -Recurse -Force -ErrorAction SilentlyContinue + if (-not ($PSBoundParameters.ContainsKey('CGManifestPath'))) + { + Write-Verbose -Verbose "New-GlobalToolNupkgFromSource: CGManifest file path not provided." + return + } + Write-Log "New-GlobalToolNupkgFromSource: Removing GlobalTool CGManifest source directory: $CGManifestPath" if (! (Test-Path -Path $CGManifestPath)) { diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1 index e2b446cb7b7..e5e1610bc2f 100644 --- a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1 +++ b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1 @@ -6,9 +6,9 @@ Import-Module "$PSScriptRoot\dockerInstall.psm1" # which was large and unstable in docker function Install-WixZip { - param($zipPath) + param($zipPath, $arm64 = $false) - $targetRoot = "${env:ProgramFiles(x86)}\WiX Toolset xcopy" + $targetRoot = $arm64 ? "${env:ProgramFiles(x86)}\Arm Support WiX Toolset xcopy" : "${env:ProgramFiles(x86)}\WiX Toolset xcopy" $binPath = Join-Path -Path $targetRoot -ChildPath 'bin' Write-Verbose "Expanding $zipPath to $binPath ..." -Verbose Expand-Archive -Path $zipPath -DestinationPath $binPath -Force