Merge remote-tracking branch 'upstream/master' into pegenvm

python · pablogsal · May 26, 2020 · May 27, 2020 · May 27, 2020 · May 27, 2020
commit 2a138cc51559f543de395d482baaceb8df430c0a
diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
@@ -1,18 +1,14 @@
 variables:
-  manylinux: false
   coverage: false
 
-resources:
-  containers:
-  - container: manylinux1
-    image: pyca/cryptography-manylinux1:x86_64
+trigger: ['master', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -24,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./docs-steps.yml
@@ -56,50 +52,19 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml
     parameters:
       dependencies: apt
 
 
-- job: ManyLinux1_CI_Tests
-  displayName: ManyLinux1 CI Tests
-  dependsOn: Prebuild
-  condition: |
-    and(
-        and(
-            succeeded(),
-            eq(variables['manylinux'], 'true')
-        ),
-        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
-    )
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  container: manylinux1
-
-  variables:
-    testRunTitle: '$(build.sourceBranchName)-manylinux1'
-    testRunPlatform: manylinux1
-    openssl_version: ''
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: yum
-      sudo_dependencies: ''
-      xvfb: false
-      patchcheck: false
-
-
 - job: Ubuntu_Coverage_CI_Tests
   displayName: Ubuntu CI Tests (coverage)
   dependsOn: Prebuild
@@ -113,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml

diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
@@ -1,18 +1,14 @@
 variables:
-  manylinux: false
   coverage: false
 
-resources:
-  containers:
-  - container: manylinux1
-    image: pyca/cryptography-manylinux1:x86_64
+pr: ['master', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -24,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   steps:
   - template: ./docs-steps.yml
@@ -56,50 +52,19 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml
     parameters:
       dependencies: apt
 
 
-- job: ManyLinux1_PR_Tests
-  displayName: ManyLinux1 PR Tests
-  dependsOn: Prebuild
-  condition: |
-    and(
-        and(
-            succeeded(),
-            eq(variables['manylinux'], 'true')
-        ),
-        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
-    )
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  container: manylinux1
-
-  variables:
-    testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1'
-    testRunPlatform: manylinux1
-    openssl_version: ''
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: yum
-      sudo_dependencies: ''
-      xvfb: false
-      patchcheck: false
-
-
 - job: Ubuntu_Coverage_PR_Tests
   displayName: Ubuntu PR Tests (coverage)
   dependsOn: Prebuild
@@ -113,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-18.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml

diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml
@@ -1,6 +1,12 @@
 steps:
   - template: ./checkout.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: doc'
     inputs:

diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml
@@ -105,9 +105,15 @@ jobs:
     clean: all
 
   steps:
-  - checkout: none
+  - template: ./checkout.yml
   - template: ./find-sdk.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - task: DownloadBuildArtifacts@0
     displayName: 'Download Artifact: unsigned_msix'
     inputs:

diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml
@@ -26,6 +26,12 @@ jobs:
   - template: ./checkout.yml
   - template: ./find-sdk.yml
 
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)"
+    displayName: 'Update signing description'
+    condition: and(succeeded(), not(variables['SigningDescription']))
+
   - powershell: |
       Write-Host "##vso[build.addbuildtag]signed"
     displayName: 'Add build tags'

diff --git a/.gitattributes b/.gitattributes
@@ -41,8 +41,6 @@ PC/readme.txt text eol=crlf
 
 # Generated files
 # https://github.com/github/linguist#generated-code
-Include/graminit.h          linguist-generated=true
-Python/graminit.h           linguist-generated=true
 Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -72,10 +72,10 @@ Include/pytime.h              @pganssle @abalkin
 /Modules/gcmodule.c           @pablogsal
 /Doc/library/gc.rst           @pablogsal
 
-# Parser/Pgen
-/Parser/pgen/                 @pablogsal
-/Parser/pegen/                @pablogsal
-/Tools/peg_generator/         @pablogsal
+# Parser
+/Parser/                      @pablogsal @lysnikolaou
+/Tools/peg_generator/         @pablogsal @lysnikolaou
+/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 
 # SQLite 3
 **/*sqlite*                   @berkerpeksag

diff --git a/.github/problem-matchers/gcc.json b/.github/problem-matchers/gcc.json
@@ -0,0 +1,18 @@
+{
+    "__comment": "Taken from vscode-cpptools's Extension/package.json gcc rule",
+    "problemMatcher": [
+        {
+            "owner": "gcc-problem-matcher",
+            "pattern": [
+                {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "severity": 4,
+                    "message": 5
+                }
+            ]
+        }
+    ]
+}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -32,7 +32,20 @@ jobs:
             echo '::set-output name=run_tests::true'
           else
             git fetch origin $GITHUB_BASE_REF --depth=1
-            git diff --name-only origin/$GITHUB_BASE_REF... | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true
+            # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more
+            # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots),
+            # but it requires to download more commits (this job uses
+            # "git fetch --depth=1").
+            #
+            # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git
+            # 2.26, but Git 2.28 is stricter and fails with "no merge base".
+            #
+            # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on
+            # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF
+            # into the PR branch anyway.
+            #
+            # https://github.com/python/core-workflow/issues/373
+            git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true
           fi
   build_win32:
     name: 'Windows (x86)'
@@ -87,6 +100,8 @@ jobs:
       OPENSSL_VER: 1.1.1f
     steps:
     - uses: actions/checkout@v2
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: 'Restore OpenSSL build'

diff --git a/.travis.yml b/.travis.yml
@@ -82,6 +82,12 @@ matrix:
           packages:
             - xvfb
       before_script:
+        - |
+            if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]
+            then
+              echo "Don't run Python coverage on pull requests."
+              exit
+            fi
         - ./configure
         - make -j4
         # Need a venv that can parse covered code.
@@ -109,6 +115,12 @@ matrix:
             - lcov
             - xvfb
       before_script:
+        - |
+            if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]
+            then
+              echo "Don't run C coverage on pull requests."
+              exit
+            fi
         - ./configure
       script:
         - xvfb-run make -j4 coverage-report

diff --git a/Doc/Makefile b/Doc/Makefile
@@ -143,7 +143,7 @@ clean:
 venv:
 	$(PYTHON) -m venv $(VENVDIR)
 	$(VENVDIR)/bin/python3 -m pip install -U pip setuptools
-	$(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.2.0 blurb python-docs-theme
+	$(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.3.1 blurb python-docs-theme
 	@echo "The venv has been created in the $(VENVDIR) directory"
 
 dist:

diff --git a/Doc/c-api/abstract.rst b/Doc/c-api/abstract.rst
@@ -24,4 +24,3 @@ but whose items have not been set to some non-\ ``NULL`` value yet.
    mapping.rst
    iter.rst
    buffer.rst
-   objbuffer.rst