From 935b352171e5966260b60176c39deed588b35c45 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 1 Nov 2018 16:24:05 -0700 Subject: [PATCH 1/5] Enable pipeline to sync PSGallery modules to AzArtifacts feed --- .../AzArtifactFeed/PSGalleryToAzArtifacts.yml | 32 ++++++++ .../SyncGalleryToAzArtifacts.psm1 | 78 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml create mode 100644 tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml new file mode 100644 index 00000000000..6ec22367f74 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml @@ -0,0 +1,32 @@ +# Sync packages from PSGallery to Azure DevOps Artifacts feed + +resources: +- repo: self + clean: true + +queue: + name: Hosted VS2017 +steps: + - powershell: | + Install-Module -Name PowerShellGet -MinimumVersion 2.0.1 -Force + Import-Module PowerShellGet -Force -Verbose + displayName: Update PSGet and PackageManagement + condition: succeededOrFailed() + + - powershell: | + Import-Module -Force "$(Build.SourcesDirectory)/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1" + SyncGalleryToAzArtifacts -AzDevOpsPAT $(AzDevOpsPAT) -Destination $(Build.ArtifactStagingDirectory) + displayName: Download packages from PSGallery that need to be updated + condition: succeededOrFailed() + + - powershell: | + Write-Verbose -Verbose "Packages to upload" + (Get-ChildItem "$(Build.ArtifactStagingDirectory)/*.nupkg").FullName + displayName: List packages to upload + condition: succeededOrFailed() + + - task: NuGetCommand@2 + displayName: 'NuGet push' + inputs: + command: push + publishVstsFeed: '366dbb95-b0d7-46d9-a50b-d1e6939d4f66' diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 new file mode 100644 index 00000000000..0f6eba6ab5e --- /dev/null +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 @@ -0,0 +1,78 @@ +function SyncGalleryToAzArtifacts { + param( + [Parameter(Mandatory = $true)] [string] $AzDevOpsPAT, + [Parameter(Mandatory = $true)] [string] $Destination + ) + + $csproj = [xml] (Get-Content 'src/Modules/PSGalleryModules.csproj') + $packages = @($csproj.Project.ItemGroup.PackageReference | ForEach-Object { [ordered] @{Name = $_.Include; Version = $_.Version }}) + + $galleryPackages = @() + $azArtifactsPackages = @() + $modulesToUpdate = @() + + $galleryUrl = 'https://www.powershellgallery.com/api/v2/' + $azArtifactsUrl = 'https://mscodehub.pkgs.visualstudio.com/_packaging/pscore-release/nuget/v2' + + $azDevOpsCreds = [pscredential]::new($env:AzDevOpsUserName, (ConvertTo-SecureString -String $AzDevOpsPAT -AsPlainText -Force)) + + foreach ($p in $packages) { + try { + # Get module from gallery + $foundPackageOnGallery = Find-Package -ProviderName NuGet -Source $galleryUrl -AllVersions -Name $p.Name -Force -AllowPreReleaseVersion | Sort-Object -Property Version -Descending | Select-Object -First 1 + Write-Verbose -Verbose "Found module $($p.Name) - $($foundPackageOnGallery.Version) in gallery" + $galleryPackages += $foundPackageOnGallery + } catch { + if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') { + # Log and ignore failure is required version is not found on gallery. + Write-Warning "Module not found on gallery $($p.Name) - $($p.Version)" + } + else { + Write-Error $_ + } + } + + try { + # Get module from Az Artifacts + # There seems to be a bug in the feed with RequiredVersion matching. Adding workaround with post filtering. + # Issue: https://github.com/OneGet/oneget/issues/397 + $foundPackageOnAz = Find-Package -ProviderName NuGet -Source $azArtifactsUrl -AllVersions -Name $p.Name -Force -Credential $azDevOpsCreds -AllowPreReleaseVersion | Sort-Object -Property Version -Descending | Select-Object -First 1 + Write-Verbose -Verbose "Found module $($p.Name) - $($foundPackageOnAz.Version) in azArtifacts" + $azArtifactsPackages += $foundPackageOnAz + } catch { + if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') { + # Log and add the module to update list. + Write-Verbose -Verbose "Az Artifacts Module needs update to - $($p.Name) - $($p.Version)" + $modulesToUpdate += $p + } + else { + Write-Error $_ + } + } + + # Check if Az package version is less that gallery version + if ($foundPackageOnAz.Version -lt $foundPackageOnGallery.Version) { + Write-Verbose -Verbose "Module needs to be updated $($p.Name) - $($foundPackageOnGallery.Version)" + $modulesToUpdate += $foundPackageOnGallery + } elseif ($foundPackageOnGallery.Version -lt $foundPackageOnAz.Version) { + Write-Warning "Newer version found on Az Artifacts - $($foundPackageOnAz.Name) - $($foundPackageOnAz.Version)" + } else { + Write-Verbose -Verbose "Module is in sync - $($p.Name)" + } + } + + Write-Verbose -Verbose "Gallery Packages:" + $galleryPackages + + Write-Verbose -Verbose "Az Artifacts Packages:" + $azArtifactsPackages + + Write-Verbose -Verbose "Modules to update:" + $modulesToUpdate + + foreach ($p in $modulesToUpdate) { + Save-Package -Provider NuGet -Source $galleryUrl -Name $p.Name -RequiredVersion $p.Version -Path $Destination + } +} + +Export-ModuleMember -Function 'SyncGalleryToAzArtifacts' From eba8e339613165842edc8c4e5e9d22a29df7b96b Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 19 Nov 2018 14:09:00 -0800 Subject: [PATCH 2/5] Add copyright header --- .../azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 index 0f6eba6ab5e..4f30a24137c 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + function SyncGalleryToAzArtifacts { param( [Parameter(Mandatory = $true)] [string] $AzDevOpsPAT, From 93e293dc8230071e142f21baecc9ebec62eb63de Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 20 Nov 2018 14:29:54 -0800 Subject: [PATCH 3/5] Remove dependent package if already present on AzDevOps Artifacts feed --- .../SyncGalleryToAzArtifacts.psm1 | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 index 4f30a24137c..e5185af773e 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 @@ -76,6 +76,28 @@ function SyncGalleryToAzArtifacts { foreach ($p in $modulesToUpdate) { Save-Package -Provider NuGet -Source $galleryUrl -Name $p.Name -RequiredVersion $p.Version -Path $Destination } + + # Remove dependent packages downloaded by Save-Module if there are already present in AzArtifacts feed. + try { + Register-PackageSource -Name local -Location $Destination -ProviderName NuGet -Force + $packageNamesToKeep = @() + $savedPackages = Find-Package -Source local -AllVersions -AllowPreReleaseVersion + + foreach($p in $savedPackages) { + $foundMatch = $azArtifactsPackages | Where-Object { $_.Name -eq $p.Name -and $_.Version -eq $p.Version } + + if(-not $foundMatch) { + Write-Verbose "Keeping package $($p.PackageFileName)" + $packageNamesToKeep += $p.PackageFilename + } + } + + Remove-Item -Path $Destination -Exclude $packageNamesToKeep -Recurse -Force + } + finally { + Unregister-PackageSource -Name local -Force -ErrorAction SilentlyContinue + } + } Export-ModuleMember -Function 'SyncGalleryToAzArtifacts' From c49cdb681d47d7ff140cba903516486533e096f0 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 26 Nov 2018 11:08:49 -0800 Subject: [PATCH 4/5] Use service connection to authenticate to the feed --- .../azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml | 5 +++-- .../AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml index 6ec22367f74..997f6030ab0 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml @@ -21,7 +21,7 @@ steps: - powershell: | Write-Verbose -Verbose "Packages to upload" - (Get-ChildItem "$(Build.ArtifactStagingDirectory)/*.nupkg").FullName + if(Test-Path $(Build.ArtifactStagingDirectory)) { Get-ChildItem "$(Build.ArtifactStagingDirectory)/*.nupkg" | ForEach-Object { $_.FullName }} displayName: List packages to upload condition: succeededOrFailed() @@ -29,4 +29,5 @@ steps: displayName: 'NuGet push' inputs: command: push - publishVstsFeed: '366dbb95-b0d7-46d9-a50b-d1e6939d4f66' + publishVstsFeed: 'https://mscodehub.pkgs.visualstudio.com/_packaging/pscore-release/nuget/v3/index.json' + publishFeedCredentials: 'AzArtifactsFeed' diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 index e5185af773e..409a91f671a 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 @@ -64,13 +64,13 @@ function SyncGalleryToAzArtifacts { } } - Write-Verbose -Verbose "Gallery Packages:" + "Gallery Packages:`n" $galleryPackages - Write-Verbose -Verbose "Az Artifacts Packages:" + "Az Artifacts Packages:`n" $azArtifactsPackages - Write-Verbose -Verbose "Modules to update:" + "Modules to update:`n" $modulesToUpdate foreach ($p in $modulesToUpdate) { From 7139efb8daf53846d4e58e75a17f88f428f31f7f Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 26 Nov 2018 15:49:19 -0800 Subject: [PATCH 5/5] Address CR feedback --- .../SyncGalleryToAzArtifacts.psm1 | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 index 409a91f671a..a75c2af9052 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1 @@ -1,6 +1,17 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +<# +.SYNOPSIS +Downloads to packages from PowerShell Gallery which are missing from the Azure DevOps Artifacts feed. + +.PARAMETER AzureDevOpsPAT +PAT for the username used for authenticating to the Azure DevOps Artifacts feed. + +.PARAMETER Destination +Path to the folder where the packages should be stored for uploading to Azure DevOps Artifacts feed. + +#> function SyncGalleryToAzArtifacts { param( [Parameter(Mandatory = $true)] [string] $AzDevOpsPAT, @@ -19,16 +30,16 @@ function SyncGalleryToAzArtifacts { $azDevOpsCreds = [pscredential]::new($env:AzDevOpsUserName, (ConvertTo-SecureString -String $AzDevOpsPAT -AsPlainText -Force)) - foreach ($p in $packages) { + foreach ($package in $packages) { try { # Get module from gallery - $foundPackageOnGallery = Find-Package -ProviderName NuGet -Source $galleryUrl -AllVersions -Name $p.Name -Force -AllowPreReleaseVersion | Sort-Object -Property Version -Descending | Select-Object -First 1 - Write-Verbose -Verbose "Found module $($p.Name) - $($foundPackageOnGallery.Version) in gallery" + $foundPackageOnGallery = Find-Package -ProviderName NuGet -Source $galleryUrl -AllVersions -Name $package.Name -Force -AllowPreReleaseVersion | Sort-Object -Property Version -Descending | Select-Object -First 1 + Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnGallery.Version) in gallery" $galleryPackages += $foundPackageOnGallery } catch { if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') { # Log and ignore failure is required version is not found on gallery. - Write-Warning "Module not found on gallery $($p.Name) - $($p.Version)" + Write-Warning "Module not found on gallery $($package.Name) - $($package.Version)" } else { Write-Error $_ @@ -39,14 +50,14 @@ function SyncGalleryToAzArtifacts { # Get module from Az Artifacts # There seems to be a bug in the feed with RequiredVersion matching. Adding workaround with post filtering. # Issue: https://github.com/OneGet/oneget/issues/397 - $foundPackageOnAz = Find-Package -ProviderName NuGet -Source $azArtifactsUrl -AllVersions -Name $p.Name -Force -Credential $azDevOpsCreds -AllowPreReleaseVersion | Sort-Object -Property Version -Descending | Select-Object -First 1 - Write-Verbose -Verbose "Found module $($p.Name) - $($foundPackageOnAz.Version) in azArtifacts" + $foundPackageOnAz = Find-Package -ProviderName NuGet -Source $azArtifactsUrl -AllVersions -Name $package.Name -Force -Credential $azDevOpsCreds -AllowPreReleaseVersion | Sort-Object -Property Version -Descending | Select-Object -First 1 + Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnAz.Version) in azArtifacts" $azArtifactsPackages += $foundPackageOnAz } catch { if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') { # Log and add the module to update list. - Write-Verbose -Verbose "Az Artifacts Module needs update to - $($p.Name) - $($p.Version)" - $modulesToUpdate += $p + Write-Verbose -Verbose "Az Artifacts Module needs update to - $($package.Name) - $($package.Version)" + $modulesToUpdate += $package } else { Write-Error $_ @@ -55,12 +66,12 @@ function SyncGalleryToAzArtifacts { # Check if Az package version is less that gallery version if ($foundPackageOnAz.Version -lt $foundPackageOnGallery.Version) { - Write-Verbose -Verbose "Module needs to be updated $($p.Name) - $($foundPackageOnGallery.Version)" + Write-Verbose -Verbose "Module needs to be updated $($package.Name) - $($foundPackageOnGallery.Version)" $modulesToUpdate += $foundPackageOnGallery } elseif ($foundPackageOnGallery.Version -lt $foundPackageOnAz.Version) { Write-Warning "Newer version found on Az Artifacts - $($foundPackageOnAz.Name) - $($foundPackageOnAz.Version)" } else { - Write-Verbose -Verbose "Module is in sync - $($p.Name)" + Write-Verbose -Verbose "Module is in sync - $($package.Name)" } } @@ -73,8 +84,8 @@ function SyncGalleryToAzArtifacts { "Modules to update:`n" $modulesToUpdate - foreach ($p in $modulesToUpdate) { - Save-Package -Provider NuGet -Source $galleryUrl -Name $p.Name -RequiredVersion $p.Version -Path $Destination + foreach ($package in $modulesToUpdate) { + Save-Package -Provider NuGet -Source $galleryUrl -Name $package.Name -RequiredVersion $package.Version -Path $Destination } # Remove dependent packages downloaded by Save-Module if there are already present in AzArtifacts feed. @@ -83,12 +94,12 @@ function SyncGalleryToAzArtifacts { $packageNamesToKeep = @() $savedPackages = Find-Package -Source local -AllVersions -AllowPreReleaseVersion - foreach($p in $savedPackages) { - $foundMatch = $azArtifactsPackages | Where-Object { $_.Name -eq $p.Name -and $_.Version -eq $p.Version } + foreach($package in $savedPackages) { + $foundMatch = $azArtifactsPackages | Where-Object { $_.Name -eq $package.Name -and $_.Version -eq $package.Version } if(-not $foundMatch) { - Write-Verbose "Keeping package $($p.PackageFileName)" - $packageNamesToKeep += $p.PackageFilename + Write-Verbose "Keeping package $($package.PackageFileName)" + $packageNamesToKeep += $package.PackageFilename } }