New

.azure-pipelines/ci.yml

@@ -57,7 +57,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
@@ -83,7 +83,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
@@ -104,7 +104,7 @@ jobs:
     matrix:
       win32:
         arch: win32
-        buildOpt:
+        buildOpt: '-p Win32'
         testRunTitle: '$(Build.SourceBranchName)-win32'
         testRunPlatform: win32
       win64:

.azure-pipelines/docs-steps.yml

@@ -12,11 +12,12 @@ steps:
   inputs:
     versionSpec: '>=3.6'
 
-- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme
+- script: python -m pip install -r requirements.txt
+  workingDirectory: '$(build.sourcesDirectory)/Doc'
   displayName: 'Install build dependencies'
 
 - ${{ if ne(parameters.latex, 'true') }}:
-  - script: make check suspicious html PYTHON=python
+  - script: make check html PYTHON=python
     workingDirectory: '$(build.sourcesDirectory)/Doc'
     displayName: 'Build documentation'
 
@@ -31,7 +32,7 @@ steps:
 - ${{ if eq(parameters.upload, 'true') }}:
   - task: PublishBuildArtifacts@1
     displayName: 'Publish docs'
-  
+
     inputs:
       PathToPublish: '$(build.sourcesDirectory)/Doc/build'
       ArtifactName: docs

.azure-pipelines/pr.yml

@@ -57,7 +57,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
@@ -83,7 +83,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1g
+    openssl_version: 1.1.1k
 
   steps:
   - template: ./posix-steps.yml
@@ -104,7 +104,7 @@ jobs:
     matrix:
       win32:
         arch: win32
-        buildOpt:
+        buildOpt: '-p Win32'
         testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
         testRunPlatform: win32
       win64:

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -120,10 +120,11 @@ jobs:
       artifactName: unsigned_msix
       downloadPath: $(Build.BinariesDirectory)
 
+  # MSIX must be signed and timestamped simultaneously
   - powershell: |
       $failed = $true
       foreach ($retry in 1..3) {
-          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix)
+          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix)
           if ($?) {
               $failed = $false
               break

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmImage: windows-2019
+    name: 'Windows Release'
 
   workspace:
     clean: all
@@ -36,6 +36,14 @@ jobs:
       nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive
     displayName: 'Create nuget package'
 
+  - powershell: |
+      gci *.nupkg | %{
+        nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite
+      }
+    displayName: 'Sign nuget package'
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    condition: and(succeeded(), variables['SigningCertificate'])
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: nuget'
     inputs:

.azure-pipelines/windows-release/stage-sign.yml

@@ -57,7 +57,7 @@ jobs:
       $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }})
       $failed = $true
       foreach ($retry in 1..10) {
-          signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files
+          signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files
           if ($?) {
               $failed = $false
               break

.gitattributes

@@ -47,7 +47,7 @@ PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
 Python/importlib.h          linguist-generated=true
 Python/importlib_external.h linguist-generated=true
-Include/Python-ast.h        linguist-generated=true
+Include/internal/pycore_ast.h   linguist-generated=true
 Python/Python-ast.c         linguist-generated=true
 Include/opcode.h            linguist-generated=true
 Python/opcode_targets.h     linguist-generated=true

.github/CODEOWNERS

@@ -19,6 +19,7 @@ Objects/frameobject.c         @markshannon
 Objects/call.c                @markshannon
 Python/ceval.c                @markshannon
 Python/compile.c              @markshannon
+Python/ast_opt.c              @isidentical
 
 # Hashing
 **/*hashlib*                  @python/crypto-team @tiran
@@ -84,6 +85,16 @@ Include/pytime.h              @pganssle @abalkin
 /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 /Grammar/python.gram          @pablogsal @lysnikolaou
 
+# AST
+Python/ast.c                  @isidentical
+Parser/asdl.py                @isidentical
+Parser/asdl_c.py              @isidentical
+Lib/ast.py                    @isidentical
+
+# Mock
+/Lib/unittest/mock.py         @cjw296
+/Lib/unittest/test/testmock/* @cjw296
+
 # SQLite 3
 **/*sqlite*                   @berkerpeksag
 

.github/problem-matchers/msvc.json

@@ -0,0 +1,19 @@
+{
+    "__comment": "Taken from vscode's vs/workbench/contrib/tasks/common/problemMatcher.ts msCompile rule",
+    "problemMatcher": [
+      {
+        "owner": "msvc-problem-matcher",
+        "pattern": [
+          {
+            "regexp": "^(?:\\s+\\d+\\>)?([^\\s].*)\\((\\d+),?(\\d+)?(?:,\\d+,\\d+)?\\)\\s*:\\s+(error|warning|info)\\s+(\\w{1,2}\\d+)\\s*:\\s*(.*)$",
+            "file": 1,
+            "line": 2,
+            "column": 3,
+            "severity": 4,
+            "code": 5,
+            "message": 6
+          }
+        ]
+      }
+    ]
+  }

.github/workflows/build.yml

@@ -23,13 +23,15 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       run_tests: ${{ steps.check.outputs.run_tests }}
+      run_ssl_tests: ${{ steps.check.outputs.run_ssl_tests }}
     steps:
       - uses: actions/checkout@v2
       - name: Check for source changes
         id: check
         run: |
-          if [ -z "GITHUB_BASE_REF" ]; then
+          if [ -z "$GITHUB_BASE_REF" ]; then
             echo '::set-output name=run_tests::true'
+            echo '::set-output name=run_ssl_tests::true'
           else
             git fetch origin $GITHUB_BASE_REF --depth=1
             # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more
@@ -46,7 +48,40 @@ jobs:
             #
             # https://github.com/python/core-workflow/issues/373
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true
+            git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo '::set-output name=run_ssl_tests::true' || true
           fi
+
+  check_generated_files:
+    name: 'Check if generated files are up to date'
+    runs-on: ubuntu-latest
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - name: Install Dependencies
+        run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Build CPython
+        run: |
+          # Build Python with the libpython dynamic library
+          ./configure --with-pydebug --enable-shared
+          make -j4 regen-all
+          make regen-stdlib-module-names
+      - name: Check for changes
+        run: |
+          changes=$(git status --porcelain)
+          # Check for changes in regenerated files
+          if ! test -z "$changes"
+          then
+            echo "Generated files not up to date. Perhaps you forgot to run make regen-all ;)"
+            echo "$changes"
+            exit 1
+          fi
+      - name: Check exported libpython symbols
+        run: make smelly
+      - name: Check limited ABI symbols
+        run: make check-limited-abi
+
   build_win32:
     name: 'Windows (x86)'
     runs-on: windows-latest
@@ -59,7 +94,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p Win32 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
 
   build_win_amd64:
     name: 'Windows (x64)'
@@ -68,12 +103,14 @@ jobs:
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
     - uses: actions/checkout@v2
+    - name: Register MSVC problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
     - name: Build CPython
       run: .\PCbuild\build.bat -e -p x64
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
 
   build_macos:
     name: 'macOS'
@@ -93,31 +130,89 @@ jobs:
 
   build_ubuntu:
     name: 'Ubuntu'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1f
+      OPENSSL_VER: 1.1.1k
     steps:
     - uses: actions/checkout@v2
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v2.1.2
+      uses: actions/cache@v2.1.4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
-      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/ccache-action@v1
     - name: Configure CPython
-      run: ./configure --with-pydebug --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER
+      run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR
     - name: Build CPython
       run: make -j4
     - name: Display build info
       run: make pythoninfo
     - name: Tests
       run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+
+  build_ubuntu_ssltests:
+    name: 'Ubuntu SSL tests with OpenSSL'
+    runs-on: ubuntu-20.04
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_ssl_tests == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        openssl_ver: [1.1.1k, 3.0.0-alpha15]
+    env:
+      OPENSSL_VER: ${{ matrix.openssl_ver }}
+      MULTISSL_DIR: ${{ github.workspace }}/multissl
+      OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
+      LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
+    steps:
+    - uses: actions/checkout@v2
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v2.1.4
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/ccache-action@v1
+    - name: Configure CPython
+      run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: SSL tests
+      run: ./python Lib/test/ssltests.py